12770 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s3; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s4; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s7; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s8; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s9; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s10; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s11; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s12; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s13 14; --------------------------------------------------------------------15; float16; --------------------------------------------------------------------17 18define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode(ptr addrspace(7) inreg %ptr, float %val) #0 {19; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:20; GFX12: ; %bb.0:21; GFX12-NEXT: s_wait_loadcnt_dscnt 0x022; GFX12-NEXT: s_wait_expcnt 0x023; GFX12-NEXT: s_wait_samplecnt 0x024; GFX12-NEXT: s_wait_bvhcnt 0x025; GFX12-NEXT: s_wait_kmcnt 0x026; GFX12-NEXT: v_mov_b32_e32 v1, s1627; GFX12-NEXT: s_wait_storecnt 0x028; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN29; GFX12-NEXT: s_wait_loadcnt 0x030; GFX12-NEXT: global_inv scope:SCOPE_DEV31; GFX12-NEXT: s_wait_loadcnt 0x032; GFX12-NEXT: s_setpc_b64 s[30:31]33;34; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:35; GFX942: ; %bb.0:36; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)37; GFX942-NEXT: v_mov_b32_e32 v1, s1638; GFX942-NEXT: buffer_wbl2 sc139; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc040; GFX942-NEXT: s_waitcnt vmcnt(0)41; GFX942-NEXT: buffer_inv sc142; GFX942-NEXT: s_setpc_b64 s[30:31]43;44; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:45; GFX11: ; %bb.0:46; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)47; GFX11-NEXT: v_mov_b32_e32 v1, s1648; GFX11-NEXT: s_waitcnt_vscnt null, 0x049; GFX11-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 glc50; GFX11-NEXT: s_waitcnt vmcnt(0)51; GFX11-NEXT: buffer_gl1_inv52; GFX11-NEXT: buffer_gl0_inv53; GFX11-NEXT: s_setpc_b64 s[30:31]54;55; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:56; GFX10: ; %bb.0:57; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)58; GFX10-NEXT: v_mov_b32_e32 v2, v059; GFX10-NEXT: v_mov_b32_e32 v0, s2060; GFX10-NEXT: v_mov_b32_e32 v3, s2061; GFX10-NEXT: s_mov_b32 s4, 062; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102463; GFX10-NEXT: .LBB0_1: ; %atomicrmw.start64; GFX10-NEXT: ; =>This Inner Loop Header: Depth=165; GFX10-NEXT: s_waitcnt vmcnt(0)66; GFX10-NEXT: v_mov_b32_e32 v5, v067; GFX10-NEXT: s_waitcnt_vscnt null, 0x068; GFX10-NEXT: v_add_f32_e32 v4, v5, v269; GFX10-NEXT: v_mov_b32_e32 v0, v470; GFX10-NEXT: v_mov_b32_e32 v1, v571; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc72; GFX10-NEXT: s_waitcnt vmcnt(0)73; GFX10-NEXT: buffer_gl1_inv74; GFX10-NEXT: buffer_gl0_inv75; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v576; GFX10-NEXT: s_or_b32 s4, vcc_lo, s477; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s478; GFX10-NEXT: s_cbranch_execnz .LBB0_179; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end80; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s481; GFX10-NEXT: s_setpc_b64 s[30:31]82;83; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:84; GFX90A: ; %bb.0:85; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)86; GFX90A-NEXT: v_mov_b32_e32 v1, s2087; GFX90A-NEXT: buffer_atomic_add_f32 v0, v1, s[16:19], 0 offen offset:1024 glc88; GFX90A-NEXT: s_waitcnt vmcnt(0)89; GFX90A-NEXT: buffer_wbinvl190; GFX90A-NEXT: s_setpc_b64 s[30:31]91;92; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:93; GFX908: ; %bb.0:94; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)95; GFX908-NEXT: v_mov_b32_e32 v2, v096; GFX908-NEXT: v_mov_b32_e32 v0, s2097; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102498; GFX908-NEXT: s_mov_b64 s[4:5], 099; GFX908-NEXT: v_mov_b32_e32 v3, s20100; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start101; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1102; GFX908-NEXT: s_waitcnt vmcnt(0)103; GFX908-NEXT: v_mov_b32_e32 v5, v0104; GFX908-NEXT: v_add_f32_e32 v4, v5, v2105; GFX908-NEXT: v_mov_b32_e32 v0, v4106; GFX908-NEXT: v_mov_b32_e32 v1, v5107; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc108; GFX908-NEXT: s_waitcnt vmcnt(0)109; GFX908-NEXT: buffer_wbinvl1110; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5111; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]112; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]113; GFX908-NEXT: s_cbranch_execnz .LBB0_1114; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end115; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]116; GFX908-NEXT: s_setpc_b64 s[30:31]117;118; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:119; GFX8: ; %bb.0:120; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)121; GFX8-NEXT: v_mov_b32_e32 v2, v0122; GFX8-NEXT: v_mov_b32_e32 v0, s20123; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024124; GFX8-NEXT: s_mov_b64 s[4:5], 0125; GFX8-NEXT: v_mov_b32_e32 v3, s20126; GFX8-NEXT: .LBB0_1: ; %atomicrmw.start127; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1128; GFX8-NEXT: s_waitcnt vmcnt(0)129; GFX8-NEXT: v_mov_b32_e32 v5, v0130; GFX8-NEXT: v_add_f32_e32 v4, v5, v2131; GFX8-NEXT: v_mov_b32_e32 v0, v4132; GFX8-NEXT: v_mov_b32_e32 v1, v5133; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc134; GFX8-NEXT: s_waitcnt vmcnt(0)135; GFX8-NEXT: buffer_wbinvl1136; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5137; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]138; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]139; GFX8-NEXT: s_cbranch_execnz .LBB0_1140; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end141; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]142; GFX8-NEXT: s_setpc_b64 s[30:31]143;144; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:145; GFX7: ; %bb.0:146; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)147; GFX7-NEXT: v_mov_b32_e32 v2, v0148; GFX7-NEXT: v_mov_b32_e32 v0, s20149; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024150; GFX7-NEXT: s_mov_b64 s[4:5], 0151; GFX7-NEXT: v_mov_b32_e32 v3, s20152; GFX7-NEXT: .LBB0_1: ; %atomicrmw.start153; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1154; GFX7-NEXT: s_waitcnt vmcnt(0)155; GFX7-NEXT: v_mov_b32_e32 v5, v0156; GFX7-NEXT: v_add_f32_e32 v4, v5, v2157; GFX7-NEXT: v_mov_b32_e32 v0, v4158; GFX7-NEXT: v_mov_b32_e32 v1, v5159; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc160; GFX7-NEXT: s_waitcnt vmcnt(0)161; GFX7-NEXT: buffer_wbinvl1162; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5163; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]164; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]165; GFX7-NEXT: s_cbranch_execnz .LBB0_1166; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end167; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]168; GFX7-NEXT: s_setpc_b64 s[30:31]169;170; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:171; GFX6: ; %bb.0:172; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)173; GFX6-NEXT: v_mov_b32_e32 v2, v0174; GFX6-NEXT: v_mov_b32_e32 v0, s20175; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024176; GFX6-NEXT: s_add_i32 s6, s20, 0x400177; GFX6-NEXT: s_mov_b64 s[4:5], 0178; GFX6-NEXT: v_mov_b32_e32 v3, s6179; GFX6-NEXT: .LBB0_1: ; %atomicrmw.start180; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1181; GFX6-NEXT: s_waitcnt vmcnt(0)182; GFX6-NEXT: v_mov_b32_e32 v5, v0183; GFX6-NEXT: v_add_f32_e32 v4, v5, v2184; GFX6-NEXT: s_waitcnt expcnt(0)185; GFX6-NEXT: v_mov_b32_e32 v0, v4186; GFX6-NEXT: v_mov_b32_e32 v1, v5187; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc188; GFX6-NEXT: s_waitcnt vmcnt(0)189; GFX6-NEXT: buffer_wbinvl1190; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5191; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]192; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]193; GFX6-NEXT: s_cbranch_execnz .LBB0_1194; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end195; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]196; GFX6-NEXT: s_waitcnt expcnt(0)197; GFX6-NEXT: s_setpc_b64 s[30:31]198 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256199 %result = atomicrmw fadd ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0200 ret float %result201}202 203define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode(ptr addrspace(7) inreg %ptr, float %val) #0 {204; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:205; GFX12: ; %bb.0:206; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0207; GFX12-NEXT: s_wait_expcnt 0x0208; GFX12-NEXT: s_wait_samplecnt 0x0209; GFX12-NEXT: s_wait_bvhcnt 0x0210; GFX12-NEXT: s_wait_kmcnt 0x0211; GFX12-NEXT: v_mov_b32_e32 v1, s16212; GFX12-NEXT: s_wait_storecnt 0x0213; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024214; GFX12-NEXT: s_wait_storecnt 0x0215; GFX12-NEXT: global_inv scope:SCOPE_DEV216; GFX12-NEXT: s_wait_loadcnt 0x0217; GFX12-NEXT: s_setpc_b64 s[30:31]218;219; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:220; GFX942: ; %bb.0:221; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)222; GFX942-NEXT: v_mov_b32_e32 v1, s16223; GFX942-NEXT: buffer_wbl2 sc1224; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024225; GFX942-NEXT: s_waitcnt vmcnt(0)226; GFX942-NEXT: buffer_inv sc1227; GFX942-NEXT: s_setpc_b64 s[30:31]228;229; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:230; GFX11: ; %bb.0:231; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)232; GFX11-NEXT: v_mov_b32_e32 v1, s16233; GFX11-NEXT: s_waitcnt_vscnt null, 0x0234; GFX11-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024235; GFX11-NEXT: s_waitcnt_vscnt null, 0x0236; GFX11-NEXT: buffer_gl1_inv237; GFX11-NEXT: buffer_gl0_inv238; GFX11-NEXT: s_setpc_b64 s[30:31]239;240; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:241; GFX10: ; %bb.0:242; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)243; GFX10-NEXT: v_mov_b32_e32 v1, s20244; GFX10-NEXT: v_mov_b32_e32 v3, s20245; GFX10-NEXT: s_mov_b32 s4, 0246; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024247; GFX10-NEXT: .LBB1_1: ; %atomicrmw.start248; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1249; GFX10-NEXT: s_waitcnt vmcnt(0)250; GFX10-NEXT: v_add_f32_e32 v1, v2, v0251; GFX10-NEXT: v_mov_b32_e32 v5, v2252; GFX10-NEXT: s_waitcnt_vscnt null, 0x0253; GFX10-NEXT: v_mov_b32_e32 v4, v1254; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc255; GFX10-NEXT: s_waitcnt vmcnt(0)256; GFX10-NEXT: buffer_gl1_inv257; GFX10-NEXT: buffer_gl0_inv258; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2259; GFX10-NEXT: v_mov_b32_e32 v2, v4260; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4261; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4262; GFX10-NEXT: s_cbranch_execnz .LBB1_1263; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end264; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4265; GFX10-NEXT: s_setpc_b64 s[30:31]266;267; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:268; GFX90A: ; %bb.0:269; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)270; GFX90A-NEXT: v_mov_b32_e32 v1, s20271; GFX90A-NEXT: buffer_atomic_add_f32 v0, v1, s[16:19], 0 offen offset:1024272; GFX90A-NEXT: s_waitcnt vmcnt(0)273; GFX90A-NEXT: buffer_wbinvl1274; GFX90A-NEXT: s_setpc_b64 s[30:31]275;276; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:277; GFX908: ; %bb.0:278; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)279; GFX908-NEXT: v_mov_b32_e32 v1, s20280; GFX908-NEXT: buffer_atomic_add_f32 v0, v1, s[16:19], 0 offen offset:1024281; GFX908-NEXT: s_waitcnt vmcnt(0)282; GFX908-NEXT: buffer_wbinvl1283; GFX908-NEXT: s_setpc_b64 s[30:31]284;285; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:286; GFX8: ; %bb.0:287; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)288; GFX8-NEXT: v_mov_b32_e32 v1, s20289; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024290; GFX8-NEXT: s_mov_b64 s[4:5], 0291; GFX8-NEXT: v_mov_b32_e32 v3, s20292; GFX8-NEXT: .LBB1_1: ; %atomicrmw.start293; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1294; GFX8-NEXT: s_waitcnt vmcnt(0)295; GFX8-NEXT: v_add_f32_e32 v1, v2, v0296; GFX8-NEXT: v_mov_b32_e32 v5, v2297; GFX8-NEXT: v_mov_b32_e32 v4, v1298; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc299; GFX8-NEXT: s_waitcnt vmcnt(0)300; GFX8-NEXT: buffer_wbinvl1301; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2302; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]303; GFX8-NEXT: v_mov_b32_e32 v2, v4304; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]305; GFX8-NEXT: s_cbranch_execnz .LBB1_1306; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end307; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]308; GFX8-NEXT: s_setpc_b64 s[30:31]309;310; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:311; GFX7: ; %bb.0:312; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)313; GFX7-NEXT: v_mov_b32_e32 v1, s20314; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024315; GFX7-NEXT: s_mov_b64 s[4:5], 0316; GFX7-NEXT: v_mov_b32_e32 v3, s20317; GFX7-NEXT: .LBB1_1: ; %atomicrmw.start318; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1319; GFX7-NEXT: s_waitcnt vmcnt(0)320; GFX7-NEXT: v_add_f32_e32 v1, v2, v0321; GFX7-NEXT: v_mov_b32_e32 v5, v2322; GFX7-NEXT: v_mov_b32_e32 v4, v1323; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc324; GFX7-NEXT: s_waitcnt vmcnt(0)325; GFX7-NEXT: buffer_wbinvl1326; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2327; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]328; GFX7-NEXT: v_mov_b32_e32 v2, v4329; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]330; GFX7-NEXT: s_cbranch_execnz .LBB1_1331; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end332; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]333; GFX7-NEXT: s_setpc_b64 s[30:31]334;335; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:336; GFX6: ; %bb.0:337; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)338; GFX6-NEXT: v_mov_b32_e32 v1, s20339; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024340; GFX6-NEXT: s_add_i32 s6, s20, 0x400341; GFX6-NEXT: s_mov_b64 s[4:5], 0342; GFX6-NEXT: v_mov_b32_e32 v3, s6343; GFX6-NEXT: .LBB1_1: ; %atomicrmw.start344; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1345; GFX6-NEXT: s_waitcnt vmcnt(0)346; GFX6-NEXT: v_add_f32_e32 v1, v2, v0347; GFX6-NEXT: s_waitcnt expcnt(0)348; GFX6-NEXT: v_mov_b32_e32 v5, v2349; GFX6-NEXT: v_mov_b32_e32 v4, v1350; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc351; GFX6-NEXT: s_waitcnt vmcnt(0)352; GFX6-NEXT: buffer_wbinvl1353; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2354; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]355; GFX6-NEXT: v_mov_b32_e32 v2, v4356; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]357; GFX6-NEXT: s_cbranch_execnz .LBB1_1358; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end359; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]360; GFX6-NEXT: s_waitcnt expcnt(0)361; GFX6-NEXT: s_setpc_b64 s[30:31]362 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256363 %unused = atomicrmw fadd ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0364 ret void365}366 367define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode(ptr addrspace(7) %ptr, float %val) #0 {368; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:369; GFX12: ; %bb.0:370; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0371; GFX12-NEXT: s_wait_expcnt 0x0372; GFX12-NEXT: s_wait_samplecnt 0x0373; GFX12-NEXT: s_wait_bvhcnt 0x0374; GFX12-NEXT: s_wait_kmcnt 0x0375; GFX12-NEXT: s_mov_b32 s1, exec_lo376; GFX12-NEXT: s_wait_storecnt 0x0377; GFX12-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1378; GFX12-NEXT: v_readfirstlane_b32 s4, v0379; GFX12-NEXT: v_readfirstlane_b32 s5, v1380; GFX12-NEXT: v_readfirstlane_b32 s6, v2381; GFX12-NEXT: v_readfirstlane_b32 s7, v3382; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)383; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)384; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]385; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]386; GFX12-NEXT: s_and_b32 s0, vcc_lo, s0387; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)388; GFX12-NEXT: s_and_saveexec_b32 s0, s0389; GFX12-NEXT: s_wait_loadcnt 0x0390; GFX12-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN391; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3392; GFX12-NEXT: ; implicit-def: $vgpr4393; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s0394; GFX12-NEXT: s_cbranch_execnz .LBB2_1395; GFX12-NEXT: ; %bb.2:396; GFX12-NEXT: s_mov_b32 exec_lo, s1397; GFX12-NEXT: s_wait_loadcnt 0x0398; GFX12-NEXT: v_mov_b32_e32 v0, v5399; GFX12-NEXT: global_inv scope:SCOPE_DEV400; GFX12-NEXT: s_wait_loadcnt 0x0401; GFX12-NEXT: s_setpc_b64 s[30:31]402;403; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:404; GFX942: ; %bb.0:405; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)406; GFX942-NEXT: s_mov_b64 s[2:3], exec407; GFX942-NEXT: buffer_wbl2 sc1408; GFX942-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1409; GFX942-NEXT: v_readfirstlane_b32 s4, v0410; GFX942-NEXT: v_readfirstlane_b32 s5, v1411; GFX942-NEXT: v_readfirstlane_b32 s6, v2412; GFX942-NEXT: v_readfirstlane_b32 s7, v3413; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]414; GFX942-NEXT: s_nop 0415; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]416; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]417; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]418; GFX942-NEXT: s_waitcnt vmcnt(0)419; GFX942-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], 0 offen offset:1024 sc0420; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3421; GFX942-NEXT: ; implicit-def: $vgpr4422; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]423; GFX942-NEXT: s_cbranch_execnz .LBB2_1424; GFX942-NEXT: ; %bb.2:425; GFX942-NEXT: s_mov_b64 exec, s[2:3]426; GFX942-NEXT: s_waitcnt vmcnt(0)427; GFX942-NEXT: v_mov_b32_e32 v0, v5428; GFX942-NEXT: buffer_inv sc1429; GFX942-NEXT: s_setpc_b64 s[30:31]430;431; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:432; GFX11: ; %bb.0:433; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)434; GFX11-NEXT: s_mov_b32 s1, exec_lo435; GFX11-NEXT: s_waitcnt_vscnt null, 0x0436; GFX11-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1437; GFX11-NEXT: v_readfirstlane_b32 s4, v0438; GFX11-NEXT: v_readfirstlane_b32 s5, v1439; GFX11-NEXT: v_readfirstlane_b32 s6, v2440; GFX11-NEXT: v_readfirstlane_b32 s7, v3441; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)442; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]443; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]444; GFX11-NEXT: s_and_b32 s0, vcc_lo, s0445; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)446; GFX11-NEXT: s_and_saveexec_b32 s0, s0447; GFX11-NEXT: s_waitcnt vmcnt(0)448; GFX11-NEXT: buffer_atomic_add_f32 v5, v4, s[4:7], 0 offen offset:1024 glc449; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3450; GFX11-NEXT: ; implicit-def: $vgpr4451; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s0452; GFX11-NEXT: s_cbranch_execnz .LBB2_1453; GFX11-NEXT: ; %bb.2:454; GFX11-NEXT: s_mov_b32 exec_lo, s1455; GFX11-NEXT: s_waitcnt vmcnt(0)456; GFX11-NEXT: v_mov_b32_e32 v0, v5457; GFX11-NEXT: buffer_gl1_inv458; GFX11-NEXT: buffer_gl0_inv459; GFX11-NEXT: s_setpc_b64 s[30:31]460;461; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:462; GFX10: ; %bb.0:463; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)464; GFX10-NEXT: s_mov_b32 s5, 0465; GFX10-NEXT: s_mov_b32 s6, exec_lo466; GFX10-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1467; GFX10-NEXT: v_readfirstlane_b32 s8, v0468; GFX10-NEXT: v_readfirstlane_b32 s9, v1469; GFX10-NEXT: v_readfirstlane_b32 s10, v2470; GFX10-NEXT: v_readfirstlane_b32 s11, v3471; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]472; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]473; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4474; GFX10-NEXT: s_and_saveexec_b32 s4, s4475; GFX10-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024476; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)477; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4478; GFX10-NEXT: s_cbranch_execnz .LBB2_1479; GFX10-NEXT: ; %bb.2:480; GFX10-NEXT: s_mov_b32 exec_lo, s6481; GFX10-NEXT: .LBB2_3: ; %atomicrmw.start482; GFX10-NEXT: ; =>This Loop Header: Depth=1483; GFX10-NEXT: ; Child Loop BB2_4 Depth 2484; GFX10-NEXT: s_waitcnt vmcnt(0)485; GFX10-NEXT: v_add_f32_e32 v7, v8, v5486; GFX10-NEXT: s_mov_b32 s6, exec_lo487; GFX10-NEXT: s_waitcnt_vscnt null, 0x0488; GFX10-NEXT: v_mov_b32_e32 v6, v7489; GFX10-NEXT: v_mov_b32_e32 v7, v8490; GFX10-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1491; GFX10-NEXT: ; => This Inner Loop Header: Depth=2492; GFX10-NEXT: v_readfirstlane_b32 s8, v0493; GFX10-NEXT: v_readfirstlane_b32 s9, v1494; GFX10-NEXT: v_readfirstlane_b32 s10, v2495; GFX10-NEXT: v_readfirstlane_b32 s11, v3496; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]497; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]498; GFX10-NEXT: s_and_b32 s4, vcc_lo, s4499; GFX10-NEXT: s_and_saveexec_b32 s4, s4500; GFX10-NEXT: s_waitcnt vmcnt(0)501; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc502; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)503; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s4504; GFX10-NEXT: s_cbranch_execnz .LBB2_4505; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1506; GFX10-NEXT: s_mov_b32 exec_lo, s6507; GFX10-NEXT: s_waitcnt vmcnt(0)508; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8509; GFX10-NEXT: v_mov_b32_e32 v8, v6510; GFX10-NEXT: buffer_gl1_inv511; GFX10-NEXT: buffer_gl0_inv512; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5513; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)514; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5515; GFX10-NEXT: s_cbranch_execnz .LBB2_3516; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end517; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5518; GFX10-NEXT: v_mov_b32_e32 v0, v6519; GFX10-NEXT: s_setpc_b64 s[30:31]520;521; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:522; GFX90A: ; %bb.0:523; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)524; GFX90A-NEXT: s_mov_b64 s[6:7], exec525; GFX90A-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1526; GFX90A-NEXT: v_readfirstlane_b32 s8, v0527; GFX90A-NEXT: v_readfirstlane_b32 s9, v1528; GFX90A-NEXT: v_readfirstlane_b32 s10, v2529; GFX90A-NEXT: v_readfirstlane_b32 s11, v3530; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]531; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]532; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]533; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]534; GFX90A-NEXT: s_waitcnt vmcnt(0)535; GFX90A-NEXT: buffer_atomic_add_f32 v5, v4, s[8:11], 0 offen offset:1024 glc536; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3537; GFX90A-NEXT: ; implicit-def: $vgpr4538; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]539; GFX90A-NEXT: s_cbranch_execnz .LBB2_1540; GFX90A-NEXT: ; %bb.2:541; GFX90A-NEXT: s_mov_b64 exec, s[6:7]542; GFX90A-NEXT: s_waitcnt vmcnt(0)543; GFX90A-NEXT: v_mov_b32_e32 v0, v5544; GFX90A-NEXT: buffer_wbinvl1545; GFX90A-NEXT: s_setpc_b64 s[30:31]546;547; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:548; GFX908: ; %bb.0:549; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)550; GFX908-NEXT: s_mov_b64 s[6:7], exec551; GFX908-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1552; GFX908-NEXT: v_readfirstlane_b32 s8, v0553; GFX908-NEXT: v_readfirstlane_b32 s9, v1554; GFX908-NEXT: v_readfirstlane_b32 s10, v2555; GFX908-NEXT: v_readfirstlane_b32 s11, v3556; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]557; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]558; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]559; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]560; GFX908-NEXT: s_nop 0561; GFX908-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024562; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]563; GFX908-NEXT: s_cbranch_execnz .LBB2_1564; GFX908-NEXT: ; %bb.2:565; GFX908-NEXT: s_mov_b64 exec, s[6:7]566; GFX908-NEXT: s_mov_b64 s[6:7], 0567; GFX908-NEXT: .LBB2_3: ; %atomicrmw.start568; GFX908-NEXT: ; =>This Loop Header: Depth=1569; GFX908-NEXT: ; Child Loop BB2_4 Depth 2570; GFX908-NEXT: s_waitcnt vmcnt(0)571; GFX908-NEXT: v_add_f32_e32 v7, v8, v5572; GFX908-NEXT: v_mov_b32_e32 v6, v7573; GFX908-NEXT: s_mov_b64 s[12:13], exec574; GFX908-NEXT: v_mov_b32_e32 v7, v8575; GFX908-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1576; GFX908-NEXT: ; => This Inner Loop Header: Depth=2577; GFX908-NEXT: v_readfirstlane_b32 s8, v0578; GFX908-NEXT: v_readfirstlane_b32 s9, v1579; GFX908-NEXT: v_readfirstlane_b32 s10, v2580; GFX908-NEXT: v_readfirstlane_b32 s11, v3581; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]582; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]583; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]584; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]585; GFX908-NEXT: s_waitcnt vmcnt(0)586; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc587; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]588; GFX908-NEXT: s_cbranch_execnz .LBB2_4589; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1590; GFX908-NEXT: s_mov_b64 exec, s[12:13]591; GFX908-NEXT: s_waitcnt vmcnt(0)592; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8593; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]594; GFX908-NEXT: v_mov_b32_e32 v8, v6595; GFX908-NEXT: buffer_wbinvl1596; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]597; GFX908-NEXT: s_cbranch_execnz .LBB2_3598; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end599; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]600; GFX908-NEXT: v_mov_b32_e32 v0, v6601; GFX908-NEXT: s_setpc_b64 s[30:31]602;603; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:604; GFX8: ; %bb.0:605; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)606; GFX8-NEXT: s_mov_b64 s[6:7], exec607; GFX8-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1608; GFX8-NEXT: v_readfirstlane_b32 s8, v0609; GFX8-NEXT: v_readfirstlane_b32 s9, v1610; GFX8-NEXT: v_readfirstlane_b32 s10, v2611; GFX8-NEXT: v_readfirstlane_b32 s11, v3612; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]613; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]614; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]615; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]616; GFX8-NEXT: s_nop 0617; GFX8-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024618; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]619; GFX8-NEXT: s_cbranch_execnz .LBB2_1620; GFX8-NEXT: ; %bb.2:621; GFX8-NEXT: s_mov_b64 exec, s[6:7]622; GFX8-NEXT: s_mov_b64 s[6:7], 0623; GFX8-NEXT: .LBB2_3: ; %atomicrmw.start624; GFX8-NEXT: ; =>This Loop Header: Depth=1625; GFX8-NEXT: ; Child Loop BB2_4 Depth 2626; GFX8-NEXT: s_waitcnt vmcnt(0)627; GFX8-NEXT: v_add_f32_e32 v7, v8, v5628; GFX8-NEXT: v_mov_b32_e32 v6, v7629; GFX8-NEXT: s_mov_b64 s[12:13], exec630; GFX8-NEXT: v_mov_b32_e32 v7, v8631; GFX8-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1632; GFX8-NEXT: ; => This Inner Loop Header: Depth=2633; GFX8-NEXT: v_readfirstlane_b32 s8, v0634; GFX8-NEXT: v_readfirstlane_b32 s9, v1635; GFX8-NEXT: v_readfirstlane_b32 s10, v2636; GFX8-NEXT: v_readfirstlane_b32 s11, v3637; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]638; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]639; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]640; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]641; GFX8-NEXT: s_waitcnt vmcnt(0)642; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc643; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]644; GFX8-NEXT: s_cbranch_execnz .LBB2_4645; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1646; GFX8-NEXT: s_mov_b64 exec, s[12:13]647; GFX8-NEXT: s_waitcnt vmcnt(0)648; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8649; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]650; GFX8-NEXT: v_mov_b32_e32 v8, v6651; GFX8-NEXT: buffer_wbinvl1652; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]653; GFX8-NEXT: s_cbranch_execnz .LBB2_3654; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end655; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]656; GFX8-NEXT: v_mov_b32_e32 v0, v6657; GFX8-NEXT: s_setpc_b64 s[30:31]658;659; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:660; GFX7: ; %bb.0:661; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)662; GFX7-NEXT: s_mov_b64 s[6:7], exec663; GFX7-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1664; GFX7-NEXT: v_readfirstlane_b32 s8, v0665; GFX7-NEXT: v_readfirstlane_b32 s9, v1666; GFX7-NEXT: v_readfirstlane_b32 s10, v2667; GFX7-NEXT: v_readfirstlane_b32 s11, v3668; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]669; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]670; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]671; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]672; GFX7-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024673; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]674; GFX7-NEXT: s_cbranch_execnz .LBB2_1675; GFX7-NEXT: ; %bb.2:676; GFX7-NEXT: s_mov_b64 exec, s[6:7]677; GFX7-NEXT: s_mov_b64 s[6:7], 0678; GFX7-NEXT: .LBB2_3: ; %atomicrmw.start679; GFX7-NEXT: ; =>This Loop Header: Depth=1680; GFX7-NEXT: ; Child Loop BB2_4 Depth 2681; GFX7-NEXT: s_waitcnt vmcnt(0)682; GFX7-NEXT: v_add_f32_e32 v7, v8, v5683; GFX7-NEXT: v_mov_b32_e32 v6, v7684; GFX7-NEXT: s_mov_b64 s[12:13], exec685; GFX7-NEXT: v_mov_b32_e32 v7, v8686; GFX7-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1687; GFX7-NEXT: ; => This Inner Loop Header: Depth=2688; GFX7-NEXT: v_readfirstlane_b32 s8, v0689; GFX7-NEXT: v_readfirstlane_b32 s9, v1690; GFX7-NEXT: v_readfirstlane_b32 s10, v2691; GFX7-NEXT: v_readfirstlane_b32 s11, v3692; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]693; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]694; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]695; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]696; GFX7-NEXT: s_waitcnt vmcnt(0)697; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc698; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]699; GFX7-NEXT: s_cbranch_execnz .LBB2_4700; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1701; GFX7-NEXT: s_mov_b64 exec, s[12:13]702; GFX7-NEXT: s_waitcnt vmcnt(0)703; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8704; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]705; GFX7-NEXT: v_mov_b32_e32 v8, v6706; GFX7-NEXT: buffer_wbinvl1707; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]708; GFX7-NEXT: s_cbranch_execnz .LBB2_3709; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end710; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]711; GFX7-NEXT: v_mov_b32_e32 v0, v6712; GFX7-NEXT: s_setpc_b64 s[30:31]713;714; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory__amdgpu_ignore_denormal_mode:715; GFX6: ; %bb.0:716; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)717; GFX6-NEXT: v_add_i32_e32 v9, vcc, 0x400, v4718; GFX6-NEXT: s_mov_b64 s[6:7], exec719; GFX6-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1720; GFX6-NEXT: v_readfirstlane_b32 s8, v0721; GFX6-NEXT: v_readfirstlane_b32 s9, v1722; GFX6-NEXT: v_readfirstlane_b32 s10, v2723; GFX6-NEXT: v_readfirstlane_b32 s11, v3724; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]725; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]726; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]727; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]728; GFX6-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024729; GFX6-NEXT: ; implicit-def: $vgpr4730; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]731; GFX6-NEXT: s_cbranch_execnz .LBB2_1732; GFX6-NEXT: ; %bb.2:733; GFX6-NEXT: s_mov_b64 exec, s[6:7]734; GFX6-NEXT: s_mov_b64 s[6:7], 0735; GFX6-NEXT: .LBB2_3: ; %atomicrmw.start736; GFX6-NEXT: ; =>This Loop Header: Depth=1737; GFX6-NEXT: ; Child Loop BB2_4 Depth 2738; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)739; GFX6-NEXT: v_add_f32_e32 v7, v8, v5740; GFX6-NEXT: v_mov_b32_e32 v6, v7741; GFX6-NEXT: s_mov_b64 s[12:13], exec742; GFX6-NEXT: v_mov_b32_e32 v7, v8743; GFX6-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1744; GFX6-NEXT: ; => This Inner Loop Header: Depth=2745; GFX6-NEXT: v_readfirstlane_b32 s8, v0746; GFX6-NEXT: v_readfirstlane_b32 s9, v1747; GFX6-NEXT: v_readfirstlane_b32 s10, v2748; GFX6-NEXT: v_readfirstlane_b32 s11, v3749; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]750; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]751; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]752; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]753; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)754; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v9, s[8:11], 0 offen glc755; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]756; GFX6-NEXT: s_cbranch_execnz .LBB2_4757; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1758; GFX6-NEXT: s_mov_b64 exec, s[12:13]759; GFX6-NEXT: s_waitcnt vmcnt(0)760; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8761; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]762; GFX6-NEXT: v_mov_b32_e32 v8, v6763; GFX6-NEXT: buffer_wbinvl1764; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]765; GFX6-NEXT: s_cbranch_execnz .LBB2_3766; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end767; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]768; GFX6-NEXT: v_mov_b32_e32 v0, v6769; GFX6-NEXT: s_waitcnt expcnt(0)770; GFX6-NEXT: s_setpc_b64 s[30:31]771 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256772 %result = atomicrmw fadd ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.ignore.denormal.mode !0773 ret float %result774}775 776define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {777; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:778; GFX12: ; %bb.0:779; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0780; GFX12-NEXT: s_wait_expcnt 0x0781; GFX12-NEXT: s_wait_samplecnt 0x0782; GFX12-NEXT: s_wait_bvhcnt 0x0783; GFX12-NEXT: s_wait_kmcnt 0x0784; GFX12-NEXT: v_mov_b32_e32 v1, s16785; GFX12-NEXT: s_wait_storecnt 0x0786; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN787; GFX12-NEXT: s_wait_loadcnt 0x0788; GFX12-NEXT: global_inv scope:SCOPE_DEV789; GFX12-NEXT: s_wait_loadcnt 0x0790; GFX12-NEXT: s_setpc_b64 s[30:31]791;792; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:793; GFX942: ; %bb.0:794; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)795; GFX942-NEXT: v_mov_b32_e32 v1, s16796; GFX942-NEXT: buffer_wbl2 sc1797; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc0798; GFX942-NEXT: s_waitcnt vmcnt(0)799; GFX942-NEXT: buffer_inv sc1800; GFX942-NEXT: s_setpc_b64 s[30:31]801;802; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:803; GFX11: ; %bb.0:804; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)805; GFX11-NEXT: v_mov_b32_e32 v1, s16806; GFX11-NEXT: s_waitcnt_vscnt null, 0x0807; GFX11-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 glc808; GFX11-NEXT: s_waitcnt vmcnt(0)809; GFX11-NEXT: buffer_gl1_inv810; GFX11-NEXT: buffer_gl0_inv811; GFX11-NEXT: s_setpc_b64 s[30:31]812;813; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:814; GFX10: ; %bb.0:815; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)816; GFX10-NEXT: v_mov_b32_e32 v2, v0817; GFX10-NEXT: v_mov_b32_e32 v0, s20818; GFX10-NEXT: v_mov_b32_e32 v3, s20819; GFX10-NEXT: s_mov_b32 s4, 0820; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024821; GFX10-NEXT: .LBB3_1: ; %atomicrmw.start822; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1823; GFX10-NEXT: s_waitcnt vmcnt(0)824; GFX10-NEXT: v_mov_b32_e32 v5, v0825; GFX10-NEXT: s_waitcnt_vscnt null, 0x0826; GFX10-NEXT: v_add_f32_e32 v4, v5, v2827; GFX10-NEXT: v_mov_b32_e32 v0, v4828; GFX10-NEXT: v_mov_b32_e32 v1, v5829; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc830; GFX10-NEXT: s_waitcnt vmcnt(0)831; GFX10-NEXT: buffer_gl1_inv832; GFX10-NEXT: buffer_gl0_inv833; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5834; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4835; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4836; GFX10-NEXT: s_cbranch_execnz .LBB3_1837; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end838; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4839; GFX10-NEXT: s_setpc_b64 s[30:31]840;841; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:842; GFX90A: ; %bb.0:843; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)844; GFX90A-NEXT: v_mov_b32_e32 v2, v0845; GFX90A-NEXT: v_mov_b32_e32 v0, s20846; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024847; GFX90A-NEXT: s_mov_b64 s[4:5], 0848; GFX90A-NEXT: v_mov_b32_e32 v3, s20849; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start850; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1851; GFX90A-NEXT: s_waitcnt vmcnt(0)852; GFX90A-NEXT: v_mov_b32_e32 v5, v0853; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2854; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]855; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc856; GFX90A-NEXT: s_waitcnt vmcnt(0)857; GFX90A-NEXT: buffer_wbinvl1858; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5859; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]860; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]861; GFX90A-NEXT: s_cbranch_execnz .LBB3_1862; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end863; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]864; GFX90A-NEXT: s_setpc_b64 s[30:31]865;866; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:867; GFX908: ; %bb.0:868; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)869; GFX908-NEXT: v_mov_b32_e32 v2, v0870; GFX908-NEXT: v_mov_b32_e32 v0, s20871; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024872; GFX908-NEXT: s_mov_b64 s[4:5], 0873; GFX908-NEXT: v_mov_b32_e32 v3, s20874; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start875; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1876; GFX908-NEXT: s_waitcnt vmcnt(0)877; GFX908-NEXT: v_mov_b32_e32 v5, v0878; GFX908-NEXT: v_add_f32_e32 v4, v5, v2879; GFX908-NEXT: v_mov_b32_e32 v0, v4880; GFX908-NEXT: v_mov_b32_e32 v1, v5881; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc882; GFX908-NEXT: s_waitcnt vmcnt(0)883; GFX908-NEXT: buffer_wbinvl1884; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5885; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]886; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]887; GFX908-NEXT: s_cbranch_execnz .LBB3_1888; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end889; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]890; GFX908-NEXT: s_setpc_b64 s[30:31]891;892; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:893; GFX8: ; %bb.0:894; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)895; GFX8-NEXT: v_mov_b32_e32 v2, v0896; GFX8-NEXT: v_mov_b32_e32 v0, s20897; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024898; GFX8-NEXT: s_mov_b64 s[4:5], 0899; GFX8-NEXT: v_mov_b32_e32 v3, s20900; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start901; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1902; GFX8-NEXT: s_waitcnt vmcnt(0)903; GFX8-NEXT: v_mov_b32_e32 v5, v0904; GFX8-NEXT: v_add_f32_e32 v4, v5, v2905; GFX8-NEXT: v_mov_b32_e32 v0, v4906; GFX8-NEXT: v_mov_b32_e32 v1, v5907; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc908; GFX8-NEXT: s_waitcnt vmcnt(0)909; GFX8-NEXT: buffer_wbinvl1910; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5911; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]912; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]913; GFX8-NEXT: s_cbranch_execnz .LBB3_1914; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end915; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]916; GFX8-NEXT: s_setpc_b64 s[30:31]917;918; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:919; GFX7: ; %bb.0:920; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)921; GFX7-NEXT: v_mov_b32_e32 v2, v0922; GFX7-NEXT: v_mov_b32_e32 v0, s20923; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024924; GFX7-NEXT: s_mov_b64 s[4:5], 0925; GFX7-NEXT: v_mov_b32_e32 v3, s20926; GFX7-NEXT: .LBB3_1: ; %atomicrmw.start927; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1928; GFX7-NEXT: s_waitcnt vmcnt(0)929; GFX7-NEXT: v_mov_b32_e32 v5, v0930; GFX7-NEXT: v_add_f32_e32 v4, v5, v2931; GFX7-NEXT: v_mov_b32_e32 v0, v4932; GFX7-NEXT: v_mov_b32_e32 v1, v5933; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc934; GFX7-NEXT: s_waitcnt vmcnt(0)935; GFX7-NEXT: buffer_wbinvl1936; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5937; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]938; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]939; GFX7-NEXT: s_cbranch_execnz .LBB3_1940; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end941; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]942; GFX7-NEXT: s_setpc_b64 s[30:31]943;944; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:945; GFX6: ; %bb.0:946; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)947; GFX6-NEXT: v_mov_b32_e32 v2, v0948; GFX6-NEXT: v_mov_b32_e32 v0, s20949; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024950; GFX6-NEXT: s_add_i32 s6, s20, 0x400951; GFX6-NEXT: s_mov_b64 s[4:5], 0952; GFX6-NEXT: v_mov_b32_e32 v3, s6953; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start954; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1955; GFX6-NEXT: s_waitcnt vmcnt(0)956; GFX6-NEXT: v_mov_b32_e32 v5, v0957; GFX6-NEXT: v_add_f32_e32 v4, v5, v2958; GFX6-NEXT: s_waitcnt expcnt(0)959; GFX6-NEXT: v_mov_b32_e32 v0, v4960; GFX6-NEXT: v_mov_b32_e32 v1, v5961; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc962; GFX6-NEXT: s_waitcnt vmcnt(0)963; GFX6-NEXT: buffer_wbinvl1964; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5965; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]966; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]967; GFX6-NEXT: s_cbranch_execnz .LBB3_1968; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end969; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]970; GFX6-NEXT: s_waitcnt expcnt(0)971; GFX6-NEXT: s_setpc_b64 s[30:31]972 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 256973 %result = atomicrmw fadd ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0974 ret float %result975}976 977define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {978; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:979; GFX12: ; %bb.0:980; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0981; GFX12-NEXT: s_wait_expcnt 0x0982; GFX12-NEXT: s_wait_samplecnt 0x0983; GFX12-NEXT: s_wait_bvhcnt 0x0984; GFX12-NEXT: s_wait_kmcnt 0x0985; GFX12-NEXT: v_mov_b32_e32 v1, s16986; GFX12-NEXT: s_wait_storecnt 0x0987; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024988; GFX12-NEXT: s_wait_storecnt 0x0989; GFX12-NEXT: global_inv scope:SCOPE_DEV990; GFX12-NEXT: s_wait_loadcnt 0x0991; GFX12-NEXT: s_setpc_b64 s[30:31]992;993; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:994; GFX942: ; %bb.0:995; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)996; GFX942-NEXT: v_mov_b32_e32 v1, s16997; GFX942-NEXT: buffer_wbl2 sc1998; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024999; GFX942-NEXT: s_waitcnt vmcnt(0)1000; GFX942-NEXT: buffer_inv sc11001; GFX942-NEXT: s_setpc_b64 s[30:31]1002;1003; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:1004; GFX11: ; %bb.0:1005; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1006; GFX11-NEXT: v_mov_b32_e32 v1, s161007; GFX11-NEXT: s_waitcnt_vscnt null, 0x01008; GFX11-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:10241009; GFX11-NEXT: s_waitcnt_vscnt null, 0x01010; GFX11-NEXT: buffer_gl1_inv1011; GFX11-NEXT: buffer_gl0_inv1012; GFX11-NEXT: s_setpc_b64 s[30:31]1013;1014; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:1015; GFX10: ; %bb.0:1016; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1017; GFX10-NEXT: v_mov_b32_e32 v1, s201018; GFX10-NEXT: v_mov_b32_e32 v3, s201019; GFX10-NEXT: s_mov_b32 s4, 01020; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10241021; GFX10-NEXT: .LBB4_1: ; %atomicrmw.start1022; GFX10-NEXT: ; =>This Inner Loop Header: Depth=11023; GFX10-NEXT: s_waitcnt vmcnt(0)1024; GFX10-NEXT: v_add_f32_e32 v1, v2, v01025; GFX10-NEXT: v_mov_b32_e32 v5, v21026; GFX10-NEXT: s_waitcnt_vscnt null, 0x01027; GFX10-NEXT: v_mov_b32_e32 v4, v11028; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc1029; GFX10-NEXT: s_waitcnt vmcnt(0)1030; GFX10-NEXT: buffer_gl1_inv1031; GFX10-NEXT: buffer_gl0_inv1032; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v21033; GFX10-NEXT: v_mov_b32_e32 v2, v41034; GFX10-NEXT: s_or_b32 s4, vcc_lo, s41035; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s41036; GFX10-NEXT: s_cbranch_execnz .LBB4_11037; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end1038; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s41039; GFX10-NEXT: s_setpc_b64 s[30:31]1040;1041; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:1042; GFX90A: ; %bb.0:1043; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1044; GFX90A-NEXT: v_mov_b32_e32 v1, s201045; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen offset:10241046; GFX90A-NEXT: s_mov_b64 s[4:5], 01047; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start1048; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11049; GFX90A-NEXT: s_waitcnt vmcnt(0)1050; GFX90A-NEXT: v_add_f32_e32 v2, v3, v01051; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]1052; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc1053; GFX90A-NEXT: s_waitcnt vmcnt(0)1054; GFX90A-NEXT: buffer_wbinvl11055; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v31056; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1057; GFX90A-NEXT: v_mov_b32_e32 v3, v41058; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1059; GFX90A-NEXT: s_cbranch_execnz .LBB4_11060; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1061; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1062; GFX90A-NEXT: s_setpc_b64 s[30:31]1063;1064; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:1065; GFX908: ; %bb.0:1066; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1067; GFX908-NEXT: v_mov_b32_e32 v1, s201068; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10241069; GFX908-NEXT: s_mov_b64 s[4:5], 01070; GFX908-NEXT: v_mov_b32_e32 v3, s201071; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start1072; GFX908-NEXT: ; =>This Inner Loop Header: Depth=11073; GFX908-NEXT: s_waitcnt vmcnt(0)1074; GFX908-NEXT: v_add_f32_e32 v1, v2, v01075; GFX908-NEXT: v_mov_b32_e32 v5, v21076; GFX908-NEXT: v_mov_b32_e32 v4, v11077; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc1078; GFX908-NEXT: s_waitcnt vmcnt(0)1079; GFX908-NEXT: buffer_wbinvl11080; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v21081; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1082; GFX908-NEXT: v_mov_b32_e32 v2, v41083; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]1084; GFX908-NEXT: s_cbranch_execnz .LBB4_11085; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end1086; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]1087; GFX908-NEXT: s_setpc_b64 s[30:31]1088;1089; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:1090; GFX8: ; %bb.0:1091; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1092; GFX8-NEXT: v_mov_b32_e32 v1, s201093; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10241094; GFX8-NEXT: s_mov_b64 s[4:5], 01095; GFX8-NEXT: v_mov_b32_e32 v3, s201096; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start1097; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11098; GFX8-NEXT: s_waitcnt vmcnt(0)1099; GFX8-NEXT: v_add_f32_e32 v1, v2, v01100; GFX8-NEXT: v_mov_b32_e32 v5, v21101; GFX8-NEXT: v_mov_b32_e32 v4, v11102; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc1103; GFX8-NEXT: s_waitcnt vmcnt(0)1104; GFX8-NEXT: buffer_wbinvl11105; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v21106; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1107; GFX8-NEXT: v_mov_b32_e32 v2, v41108; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1109; GFX8-NEXT: s_cbranch_execnz .LBB4_11110; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1111; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1112; GFX8-NEXT: s_setpc_b64 s[30:31]1113;1114; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:1115; GFX7: ; %bb.0:1116; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1117; GFX7-NEXT: v_mov_b32_e32 v1, s201118; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10241119; GFX7-NEXT: s_mov_b64 s[4:5], 01120; GFX7-NEXT: v_mov_b32_e32 v3, s201121; GFX7-NEXT: .LBB4_1: ; %atomicrmw.start1122; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11123; GFX7-NEXT: s_waitcnt vmcnt(0)1124; GFX7-NEXT: v_add_f32_e32 v1, v2, v01125; GFX7-NEXT: v_mov_b32_e32 v5, v21126; GFX7-NEXT: v_mov_b32_e32 v4, v11127; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc1128; GFX7-NEXT: s_waitcnt vmcnt(0)1129; GFX7-NEXT: buffer_wbinvl11130; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v21131; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1132; GFX7-NEXT: v_mov_b32_e32 v2, v41133; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1134; GFX7-NEXT: s_cbranch_execnz .LBB4_11135; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1136; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1137; GFX7-NEXT: s_setpc_b64 s[30:31]1138;1139; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_grained_memory:1140; GFX6: ; %bb.0:1141; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1142; GFX6-NEXT: v_mov_b32_e32 v1, s201143; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10241144; GFX6-NEXT: s_add_i32 s6, s20, 0x4001145; GFX6-NEXT: s_mov_b64 s[4:5], 01146; GFX6-NEXT: v_mov_b32_e32 v3, s61147; GFX6-NEXT: .LBB4_1: ; %atomicrmw.start1148; GFX6-NEXT: ; =>This Inner Loop Header: Depth=11149; GFX6-NEXT: s_waitcnt vmcnt(0)1150; GFX6-NEXT: v_add_f32_e32 v1, v2, v01151; GFX6-NEXT: s_waitcnt expcnt(0)1152; GFX6-NEXT: v_mov_b32_e32 v5, v21153; GFX6-NEXT: v_mov_b32_e32 v4, v11154; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc1155; GFX6-NEXT: s_waitcnt vmcnt(0)1156; GFX6-NEXT: buffer_wbinvl11157; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v21158; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1159; GFX6-NEXT: v_mov_b32_e32 v2, v41160; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]1161; GFX6-NEXT: s_cbranch_execnz .LBB4_11162; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end1163; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]1164; GFX6-NEXT: s_waitcnt expcnt(0)1165; GFX6-NEXT: s_setpc_b64 s[30:31]1166 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 2561167 %unused = atomicrmw fadd ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !01168 ret void1169}1170 1171define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7) inreg %ptr, float %val) {1172; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:1173; GFX12: ; %bb.0:1174; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01175; GFX12-NEXT: s_wait_expcnt 0x01176; GFX12-NEXT: s_wait_samplecnt 0x01177; GFX12-NEXT: s_wait_bvhcnt 0x01178; GFX12-NEXT: s_wait_kmcnt 0x01179; GFX12-NEXT: v_mov_b32_e32 v1, s161180; GFX12-NEXT: s_wait_storecnt 0x01181; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN1182; GFX12-NEXT: s_wait_loadcnt 0x01183; GFX12-NEXT: global_inv scope:SCOPE_DEV1184; GFX12-NEXT: s_wait_loadcnt 0x01185; GFX12-NEXT: s_setpc_b64 s[30:31]1186;1187; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:1188; GFX942: ; %bb.0:1189; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1190; GFX942-NEXT: v_mov_b32_e32 v1, s161191; GFX942-NEXT: buffer_wbl2 sc11192; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc01193; GFX942-NEXT: s_waitcnt vmcnt(0)1194; GFX942-NEXT: buffer_inv sc11195; GFX942-NEXT: s_setpc_b64 s[30:31]1196;1197; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:1198; GFX11: ; %bb.0:1199; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1200; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s161201; GFX11-NEXT: v_mov_b32_e32 v0, s161202; GFX11-NEXT: s_mov_b32 s4, 01203; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:10241204; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start1205; GFX11-NEXT: ; =>This Inner Loop Header: Depth=11206; GFX11-NEXT: s_waitcnt vmcnt(0)1207; GFX11-NEXT: v_mov_b32_e32 v5, v01208; GFX11-NEXT: s_waitcnt_vscnt null, 0x01209; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1210; GFX11-NEXT: v_add_f32_e32 v4, v5, v21211; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v51212; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc1213; GFX11-NEXT: s_waitcnt vmcnt(0)1214; GFX11-NEXT: buffer_gl1_inv1215; GFX11-NEXT: buffer_gl0_inv1216; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v51217; GFX11-NEXT: s_or_b32 s4, vcc_lo, s41218; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1219; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s41220; GFX11-NEXT: s_cbranch_execnz .LBB5_11221; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end1222; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s41223; GFX11-NEXT: s_setpc_b64 s[30:31]1224;1225; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:1226; GFX10: ; %bb.0:1227; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1228; GFX10-NEXT: v_mov_b32_e32 v2, v01229; GFX10-NEXT: v_mov_b32_e32 v0, s201230; GFX10-NEXT: v_mov_b32_e32 v3, s201231; GFX10-NEXT: s_mov_b32 s4, 01232; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241233; GFX10-NEXT: .LBB5_1: ; %atomicrmw.start1234; GFX10-NEXT: ; =>This Inner Loop Header: Depth=11235; GFX10-NEXT: s_waitcnt vmcnt(0)1236; GFX10-NEXT: v_mov_b32_e32 v5, v01237; GFX10-NEXT: s_waitcnt_vscnt null, 0x01238; GFX10-NEXT: v_add_f32_e32 v4, v5, v21239; GFX10-NEXT: v_mov_b32_e32 v0, v41240; GFX10-NEXT: v_mov_b32_e32 v1, v51241; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1242; GFX10-NEXT: s_waitcnt vmcnt(0)1243; GFX10-NEXT: buffer_gl1_inv1244; GFX10-NEXT: buffer_gl0_inv1245; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v51246; GFX10-NEXT: s_or_b32 s4, vcc_lo, s41247; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s41248; GFX10-NEXT: s_cbranch_execnz .LBB5_11249; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end1250; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s41251; GFX10-NEXT: s_setpc_b64 s[30:31]1252;1253; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:1254; GFX90A: ; %bb.0:1255; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1256; GFX90A-NEXT: v_mov_b32_e32 v2, v01257; GFX90A-NEXT: v_mov_b32_e32 v0, s201258; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241259; GFX90A-NEXT: s_mov_b64 s[4:5], 01260; GFX90A-NEXT: v_mov_b32_e32 v3, s201261; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start1262; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11263; GFX90A-NEXT: s_waitcnt vmcnt(0)1264; GFX90A-NEXT: v_mov_b32_e32 v5, v01265; GFX90A-NEXT: v_add_f32_e32 v4, v5, v21266; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]1267; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1268; GFX90A-NEXT: s_waitcnt vmcnt(0)1269; GFX90A-NEXT: buffer_wbinvl11270; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51271; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1272; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1273; GFX90A-NEXT: s_cbranch_execnz .LBB5_11274; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1275; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1276; GFX90A-NEXT: s_setpc_b64 s[30:31]1277;1278; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:1279; GFX908: ; %bb.0:1280; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1281; GFX908-NEXT: v_mov_b32_e32 v2, v01282; GFX908-NEXT: v_mov_b32_e32 v0, s201283; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241284; GFX908-NEXT: s_mov_b64 s[4:5], 01285; GFX908-NEXT: v_mov_b32_e32 v3, s201286; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start1287; GFX908-NEXT: ; =>This Inner Loop Header: Depth=11288; GFX908-NEXT: s_waitcnt vmcnt(0)1289; GFX908-NEXT: v_mov_b32_e32 v5, v01290; GFX908-NEXT: v_add_f32_e32 v4, v5, v21291; GFX908-NEXT: v_mov_b32_e32 v0, v41292; GFX908-NEXT: v_mov_b32_e32 v1, v51293; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1294; GFX908-NEXT: s_waitcnt vmcnt(0)1295; GFX908-NEXT: buffer_wbinvl11296; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51297; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1298; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]1299; GFX908-NEXT: s_cbranch_execnz .LBB5_11300; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end1301; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]1302; GFX908-NEXT: s_setpc_b64 s[30:31]1303;1304; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:1305; GFX8: ; %bb.0:1306; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1307; GFX8-NEXT: v_mov_b32_e32 v2, v01308; GFX8-NEXT: v_mov_b32_e32 v0, s201309; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241310; GFX8-NEXT: s_mov_b64 s[4:5], 01311; GFX8-NEXT: v_mov_b32_e32 v3, s201312; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start1313; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11314; GFX8-NEXT: s_waitcnt vmcnt(0)1315; GFX8-NEXT: v_mov_b32_e32 v5, v01316; GFX8-NEXT: v_add_f32_e32 v4, v5, v21317; GFX8-NEXT: v_mov_b32_e32 v0, v41318; GFX8-NEXT: v_mov_b32_e32 v1, v51319; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1320; GFX8-NEXT: s_waitcnt vmcnt(0)1321; GFX8-NEXT: buffer_wbinvl11322; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51323; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1324; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1325; GFX8-NEXT: s_cbranch_execnz .LBB5_11326; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1327; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1328; GFX8-NEXT: s_setpc_b64 s[30:31]1329;1330; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:1331; GFX7: ; %bb.0:1332; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1333; GFX7-NEXT: v_mov_b32_e32 v2, v01334; GFX7-NEXT: v_mov_b32_e32 v0, s201335; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241336; GFX7-NEXT: s_mov_b64 s[4:5], 01337; GFX7-NEXT: v_mov_b32_e32 v3, s201338; GFX7-NEXT: .LBB5_1: ; %atomicrmw.start1339; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11340; GFX7-NEXT: s_waitcnt vmcnt(0)1341; GFX7-NEXT: v_mov_b32_e32 v5, v01342; GFX7-NEXT: v_add_f32_e32 v4, v5, v21343; GFX7-NEXT: v_mov_b32_e32 v0, v41344; GFX7-NEXT: v_mov_b32_e32 v1, v51345; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1346; GFX7-NEXT: s_waitcnt vmcnt(0)1347; GFX7-NEXT: buffer_wbinvl11348; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51349; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1350; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1351; GFX7-NEXT: s_cbranch_execnz .LBB5_11352; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1353; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1354; GFX7-NEXT: s_setpc_b64 s[30:31]1355;1356; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:1357; GFX6: ; %bb.0:1358; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1359; GFX6-NEXT: v_mov_b32_e32 v2, v01360; GFX6-NEXT: v_mov_b32_e32 v0, s201361; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241362; GFX6-NEXT: s_add_i32 s6, s20, 0x4001363; GFX6-NEXT: s_mov_b64 s[4:5], 01364; GFX6-NEXT: v_mov_b32_e32 v3, s61365; GFX6-NEXT: .LBB5_1: ; %atomicrmw.start1366; GFX6-NEXT: ; =>This Inner Loop Header: Depth=11367; GFX6-NEXT: s_waitcnt vmcnt(0)1368; GFX6-NEXT: v_mov_b32_e32 v5, v01369; GFX6-NEXT: v_add_f32_e32 v4, v5, v21370; GFX6-NEXT: s_waitcnt expcnt(0)1371; GFX6-NEXT: v_mov_b32_e32 v0, v41372; GFX6-NEXT: v_mov_b32_e32 v1, v51373; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc1374; GFX6-NEXT: s_waitcnt vmcnt(0)1375; GFX6-NEXT: buffer_wbinvl11376; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51377; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1378; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]1379; GFX6-NEXT: s_cbranch_execnz .LBB5_11380; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end1381; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]1382; GFX6-NEXT: s_waitcnt expcnt(0)1383; GFX6-NEXT: s_setpc_b64 s[30:31]1384 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 2561385 %result = atomicrmw fadd ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst1386 ret float %result1387}1388 1389define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {1390; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:1391; GFX12: ; %bb.0:1392; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01393; GFX12-NEXT: s_wait_expcnt 0x01394; GFX12-NEXT: s_wait_samplecnt 0x01395; GFX12-NEXT: s_wait_bvhcnt 0x01396; GFX12-NEXT: s_wait_kmcnt 0x01397; GFX12-NEXT: v_mov_b32_e32 v1, s161398; GFX12-NEXT: s_wait_storecnt 0x01399; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN1400; GFX12-NEXT: s_wait_loadcnt 0x01401; GFX12-NEXT: global_inv scope:SCOPE_DEV1402; GFX12-NEXT: s_wait_loadcnt 0x01403; GFX12-NEXT: s_setpc_b64 s[30:31]1404;1405; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:1406; GFX942: ; %bb.0:1407; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1408; GFX942-NEXT: v_mov_b32_e32 v1, s161409; GFX942-NEXT: buffer_wbl2 sc11410; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc01411; GFX942-NEXT: s_waitcnt vmcnt(0)1412; GFX942-NEXT: buffer_inv sc11413; GFX942-NEXT: s_setpc_b64 s[30:31]1414;1415; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:1416; GFX11: ; %bb.0:1417; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1418; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s161419; GFX11-NEXT: v_mov_b32_e32 v0, s161420; GFX11-NEXT: s_mov_b32 s4, 01421; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:10241422; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start1423; GFX11-NEXT: ; =>This Inner Loop Header: Depth=11424; GFX11-NEXT: s_waitcnt vmcnt(0)1425; GFX11-NEXT: v_mov_b32_e32 v5, v01426; GFX11-NEXT: s_waitcnt_vscnt null, 0x01427; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1428; GFX11-NEXT: v_add_f32_e32 v4, v5, v21429; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v51430; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc1431; GFX11-NEXT: s_waitcnt vmcnt(0)1432; GFX11-NEXT: buffer_gl1_inv1433; GFX11-NEXT: buffer_gl0_inv1434; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v51435; GFX11-NEXT: s_or_b32 s4, vcc_lo, s41436; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1437; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s41438; GFX11-NEXT: s_cbranch_execnz .LBB6_11439; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end1440; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s41441; GFX11-NEXT: s_setpc_b64 s[30:31]1442;1443; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:1444; GFX10: ; %bb.0:1445; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1446; GFX10-NEXT: v_mov_b32_e32 v2, v01447; GFX10-NEXT: v_mov_b32_e32 v0, s201448; GFX10-NEXT: v_mov_b32_e32 v3, s201449; GFX10-NEXT: s_mov_b32 s4, 01450; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241451; GFX10-NEXT: .LBB6_1: ; %atomicrmw.start1452; GFX10-NEXT: ; =>This Inner Loop Header: Depth=11453; GFX10-NEXT: s_waitcnt vmcnt(0)1454; GFX10-NEXT: v_mov_b32_e32 v5, v01455; GFX10-NEXT: s_waitcnt_vscnt null, 0x01456; GFX10-NEXT: v_add_f32_e32 v4, v5, v21457; GFX10-NEXT: v_mov_b32_e32 v0, v41458; GFX10-NEXT: v_mov_b32_e32 v1, v51459; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1460; GFX10-NEXT: s_waitcnt vmcnt(0)1461; GFX10-NEXT: buffer_gl1_inv1462; GFX10-NEXT: buffer_gl0_inv1463; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v51464; GFX10-NEXT: s_or_b32 s4, vcc_lo, s41465; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s41466; GFX10-NEXT: s_cbranch_execnz .LBB6_11467; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end1468; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s41469; GFX10-NEXT: s_setpc_b64 s[30:31]1470;1471; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:1472; GFX90A: ; %bb.0:1473; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1474; GFX90A-NEXT: v_mov_b32_e32 v2, v01475; GFX90A-NEXT: v_mov_b32_e32 v0, s201476; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241477; GFX90A-NEXT: s_mov_b64 s[4:5], 01478; GFX90A-NEXT: v_mov_b32_e32 v3, s201479; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start1480; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11481; GFX90A-NEXT: s_waitcnt vmcnt(0)1482; GFX90A-NEXT: v_mov_b32_e32 v5, v01483; GFX90A-NEXT: v_add_f32_e32 v4, v5, v21484; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]1485; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1486; GFX90A-NEXT: s_waitcnt vmcnt(0)1487; GFX90A-NEXT: buffer_wbinvl11488; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51489; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1490; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1491; GFX90A-NEXT: s_cbranch_execnz .LBB6_11492; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1493; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1494; GFX90A-NEXT: s_setpc_b64 s[30:31]1495;1496; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:1497; GFX908: ; %bb.0:1498; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1499; GFX908-NEXT: v_mov_b32_e32 v2, v01500; GFX908-NEXT: v_mov_b32_e32 v0, s201501; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241502; GFX908-NEXT: s_mov_b64 s[4:5], 01503; GFX908-NEXT: v_mov_b32_e32 v3, s201504; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start1505; GFX908-NEXT: ; =>This Inner Loop Header: Depth=11506; GFX908-NEXT: s_waitcnt vmcnt(0)1507; GFX908-NEXT: v_mov_b32_e32 v5, v01508; GFX908-NEXT: v_add_f32_e32 v4, v5, v21509; GFX908-NEXT: v_mov_b32_e32 v0, v41510; GFX908-NEXT: v_mov_b32_e32 v1, v51511; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1512; GFX908-NEXT: s_waitcnt vmcnt(0)1513; GFX908-NEXT: buffer_wbinvl11514; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51515; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1516; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]1517; GFX908-NEXT: s_cbranch_execnz .LBB6_11518; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end1519; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]1520; GFX908-NEXT: s_setpc_b64 s[30:31]1521;1522; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:1523; GFX8: ; %bb.0:1524; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1525; GFX8-NEXT: v_mov_b32_e32 v2, v01526; GFX8-NEXT: v_mov_b32_e32 v0, s201527; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241528; GFX8-NEXT: s_mov_b64 s[4:5], 01529; GFX8-NEXT: v_mov_b32_e32 v3, s201530; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start1531; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11532; GFX8-NEXT: s_waitcnt vmcnt(0)1533; GFX8-NEXT: v_mov_b32_e32 v5, v01534; GFX8-NEXT: v_add_f32_e32 v4, v5, v21535; GFX8-NEXT: v_mov_b32_e32 v0, v41536; GFX8-NEXT: v_mov_b32_e32 v1, v51537; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1538; GFX8-NEXT: s_waitcnt vmcnt(0)1539; GFX8-NEXT: buffer_wbinvl11540; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51541; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1542; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1543; GFX8-NEXT: s_cbranch_execnz .LBB6_11544; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1545; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1546; GFX8-NEXT: s_setpc_b64 s[30:31]1547;1548; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:1549; GFX7: ; %bb.0:1550; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1551; GFX7-NEXT: v_mov_b32_e32 v2, v01552; GFX7-NEXT: v_mov_b32_e32 v0, s201553; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241554; GFX7-NEXT: s_mov_b64 s[4:5], 01555; GFX7-NEXT: v_mov_b32_e32 v3, s201556; GFX7-NEXT: .LBB6_1: ; %atomicrmw.start1557; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11558; GFX7-NEXT: s_waitcnt vmcnt(0)1559; GFX7-NEXT: v_mov_b32_e32 v5, v01560; GFX7-NEXT: v_add_f32_e32 v4, v5, v21561; GFX7-NEXT: v_mov_b32_e32 v0, v41562; GFX7-NEXT: v_mov_b32_e32 v1, v51563; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1564; GFX7-NEXT: s_waitcnt vmcnt(0)1565; GFX7-NEXT: buffer_wbinvl11566; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51567; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1568; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1569; GFX7-NEXT: s_cbranch_execnz .LBB6_11570; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1571; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1572; GFX7-NEXT: s_setpc_b64 s[30:31]1573;1574; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:1575; GFX6: ; %bb.0:1576; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1577; GFX6-NEXT: v_mov_b32_e32 v2, v01578; GFX6-NEXT: v_mov_b32_e32 v0, s201579; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241580; GFX6-NEXT: s_add_i32 s6, s20, 0x4001581; GFX6-NEXT: s_mov_b64 s[4:5], 01582; GFX6-NEXT: v_mov_b32_e32 v3, s61583; GFX6-NEXT: .LBB6_1: ; %atomicrmw.start1584; GFX6-NEXT: ; =>This Inner Loop Header: Depth=11585; GFX6-NEXT: s_waitcnt vmcnt(0)1586; GFX6-NEXT: v_mov_b32_e32 v5, v01587; GFX6-NEXT: v_add_f32_e32 v4, v5, v21588; GFX6-NEXT: s_waitcnt expcnt(0)1589; GFX6-NEXT: v_mov_b32_e32 v0, v41590; GFX6-NEXT: v_mov_b32_e32 v1, v51591; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc1592; GFX6-NEXT: s_waitcnt vmcnt(0)1593; GFX6-NEXT: buffer_wbinvl11594; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51595; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1596; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]1597; GFX6-NEXT: s_cbranch_execnz .LBB6_11598; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end1599; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]1600; GFX6-NEXT: s_waitcnt expcnt(0)1601; GFX6-NEXT: s_setpc_b64 s[30:31]1602 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 2561603 %result = atomicrmw fadd ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !01604 ret float %result1605}1606 1607define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode(ptr addrspace(7) inreg %ptr, float %val) #0 {1608; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:1609; GFX12: ; %bb.0:1610; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01611; GFX12-NEXT: s_wait_expcnt 0x01612; GFX12-NEXT: s_wait_samplecnt 0x01613; GFX12-NEXT: s_wait_bvhcnt 0x01614; GFX12-NEXT: s_wait_kmcnt 0x01615; GFX12-NEXT: v_mov_b32_e32 v1, s161616; GFX12-NEXT: s_wait_storecnt 0x01617; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN1618; GFX12-NEXT: s_wait_loadcnt 0x01619; GFX12-NEXT: global_inv scope:SCOPE_DEV1620; GFX12-NEXT: s_wait_loadcnt 0x01621; GFX12-NEXT: s_setpc_b64 s[30:31]1622;1623; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:1624; GFX942: ; %bb.0:1625; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1626; GFX942-NEXT: v_mov_b32_e32 v1, s161627; GFX942-NEXT: buffer_wbl2 sc11628; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc01629; GFX942-NEXT: s_waitcnt vmcnt(0)1630; GFX942-NEXT: buffer_inv sc11631; GFX942-NEXT: s_setpc_b64 s[30:31]1632;1633; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:1634; GFX11: ; %bb.0:1635; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1636; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s161637; GFX11-NEXT: v_mov_b32_e32 v0, s161638; GFX11-NEXT: s_mov_b32 s4, 01639; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:10241640; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start1641; GFX11-NEXT: ; =>This Inner Loop Header: Depth=11642; GFX11-NEXT: s_waitcnt vmcnt(0)1643; GFX11-NEXT: v_mov_b32_e32 v5, v01644; GFX11-NEXT: s_waitcnt_vscnt null, 0x01645; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1646; GFX11-NEXT: v_add_f32_e32 v4, v5, v21647; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v51648; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc1649; GFX11-NEXT: s_waitcnt vmcnt(0)1650; GFX11-NEXT: buffer_gl1_inv1651; GFX11-NEXT: buffer_gl0_inv1652; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v51653; GFX11-NEXT: s_or_b32 s4, vcc_lo, s41654; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1655; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s41656; GFX11-NEXT: s_cbranch_execnz .LBB7_11657; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end1658; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s41659; GFX11-NEXT: s_setpc_b64 s[30:31]1660;1661; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:1662; GFX10: ; %bb.0:1663; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1664; GFX10-NEXT: v_mov_b32_e32 v2, v01665; GFX10-NEXT: v_mov_b32_e32 v0, s201666; GFX10-NEXT: v_mov_b32_e32 v3, s201667; GFX10-NEXT: s_mov_b32 s4, 01668; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241669; GFX10-NEXT: .LBB7_1: ; %atomicrmw.start1670; GFX10-NEXT: ; =>This Inner Loop Header: Depth=11671; GFX10-NEXT: s_waitcnt vmcnt(0)1672; GFX10-NEXT: v_mov_b32_e32 v5, v01673; GFX10-NEXT: s_waitcnt_vscnt null, 0x01674; GFX10-NEXT: v_add_f32_e32 v4, v5, v21675; GFX10-NEXT: v_mov_b32_e32 v0, v41676; GFX10-NEXT: v_mov_b32_e32 v1, v51677; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1678; GFX10-NEXT: s_waitcnt vmcnt(0)1679; GFX10-NEXT: buffer_gl1_inv1680; GFX10-NEXT: buffer_gl0_inv1681; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v51682; GFX10-NEXT: s_or_b32 s4, vcc_lo, s41683; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s41684; GFX10-NEXT: s_cbranch_execnz .LBB7_11685; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end1686; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s41687; GFX10-NEXT: s_setpc_b64 s[30:31]1688;1689; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:1690; GFX90A: ; %bb.0:1691; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1692; GFX90A-NEXT: v_mov_b32_e32 v2, v01693; GFX90A-NEXT: v_mov_b32_e32 v0, s201694; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241695; GFX90A-NEXT: s_mov_b64 s[4:5], 01696; GFX90A-NEXT: v_mov_b32_e32 v3, s201697; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start1698; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11699; GFX90A-NEXT: s_waitcnt vmcnt(0)1700; GFX90A-NEXT: v_mov_b32_e32 v5, v01701; GFX90A-NEXT: v_add_f32_e32 v4, v5, v21702; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]1703; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1704; GFX90A-NEXT: s_waitcnt vmcnt(0)1705; GFX90A-NEXT: buffer_wbinvl11706; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51707; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1708; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1709; GFX90A-NEXT: s_cbranch_execnz .LBB7_11710; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1711; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1712; GFX90A-NEXT: s_setpc_b64 s[30:31]1713;1714; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:1715; GFX908: ; %bb.0:1716; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1717; GFX908-NEXT: v_mov_b32_e32 v2, v01718; GFX908-NEXT: v_mov_b32_e32 v0, s201719; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241720; GFX908-NEXT: s_mov_b64 s[4:5], 01721; GFX908-NEXT: v_mov_b32_e32 v3, s201722; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start1723; GFX908-NEXT: ; =>This Inner Loop Header: Depth=11724; GFX908-NEXT: s_waitcnt vmcnt(0)1725; GFX908-NEXT: v_mov_b32_e32 v5, v01726; GFX908-NEXT: v_add_f32_e32 v4, v5, v21727; GFX908-NEXT: v_mov_b32_e32 v0, v41728; GFX908-NEXT: v_mov_b32_e32 v1, v51729; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1730; GFX908-NEXT: s_waitcnt vmcnt(0)1731; GFX908-NEXT: buffer_wbinvl11732; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51733; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1734; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]1735; GFX908-NEXT: s_cbranch_execnz .LBB7_11736; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end1737; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]1738; GFX908-NEXT: s_setpc_b64 s[30:31]1739;1740; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:1741; GFX8: ; %bb.0:1742; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1743; GFX8-NEXT: v_mov_b32_e32 v2, v01744; GFX8-NEXT: v_mov_b32_e32 v0, s201745; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241746; GFX8-NEXT: s_mov_b64 s[4:5], 01747; GFX8-NEXT: v_mov_b32_e32 v3, s201748; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start1749; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11750; GFX8-NEXT: s_waitcnt vmcnt(0)1751; GFX8-NEXT: v_mov_b32_e32 v5, v01752; GFX8-NEXT: v_add_f32_e32 v4, v5, v21753; GFX8-NEXT: v_mov_b32_e32 v0, v41754; GFX8-NEXT: v_mov_b32_e32 v1, v51755; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1756; GFX8-NEXT: s_waitcnt vmcnt(0)1757; GFX8-NEXT: buffer_wbinvl11758; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51759; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1760; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1761; GFX8-NEXT: s_cbranch_execnz .LBB7_11762; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1763; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1764; GFX8-NEXT: s_setpc_b64 s[30:31]1765;1766; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:1767; GFX7: ; %bb.0:1768; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1769; GFX7-NEXT: v_mov_b32_e32 v2, v01770; GFX7-NEXT: v_mov_b32_e32 v0, s201771; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241772; GFX7-NEXT: s_mov_b64 s[4:5], 01773; GFX7-NEXT: v_mov_b32_e32 v3, s201774; GFX7-NEXT: .LBB7_1: ; %atomicrmw.start1775; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11776; GFX7-NEXT: s_waitcnt vmcnt(0)1777; GFX7-NEXT: v_mov_b32_e32 v5, v01778; GFX7-NEXT: v_add_f32_e32 v4, v5, v21779; GFX7-NEXT: v_mov_b32_e32 v0, v41780; GFX7-NEXT: v_mov_b32_e32 v1, v51781; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc1782; GFX7-NEXT: s_waitcnt vmcnt(0)1783; GFX7-NEXT: buffer_wbinvl11784; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51785; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1786; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1787; GFX7-NEXT: s_cbranch_execnz .LBB7_11788; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1789; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1790; GFX7-NEXT: s_setpc_b64 s[30:31]1791;1792; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:1793; GFX6: ; %bb.0:1794; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1795; GFX6-NEXT: v_mov_b32_e32 v2, v01796; GFX6-NEXT: v_mov_b32_e32 v0, s201797; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10241798; GFX6-NEXT: s_add_i32 s6, s20, 0x4001799; GFX6-NEXT: s_mov_b64 s[4:5], 01800; GFX6-NEXT: v_mov_b32_e32 v3, s61801; GFX6-NEXT: .LBB7_1: ; %atomicrmw.start1802; GFX6-NEXT: ; =>This Inner Loop Header: Depth=11803; GFX6-NEXT: s_waitcnt vmcnt(0)1804; GFX6-NEXT: v_mov_b32_e32 v5, v01805; GFX6-NEXT: v_add_f32_e32 v4, v5, v21806; GFX6-NEXT: s_waitcnt expcnt(0)1807; GFX6-NEXT: v_mov_b32_e32 v0, v41808; GFX6-NEXT: v_mov_b32_e32 v1, v51809; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc1810; GFX6-NEXT: s_waitcnt vmcnt(0)1811; GFX6-NEXT: buffer_wbinvl11812; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v51813; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1814; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]1815; GFX6-NEXT: s_cbranch_execnz .LBB7_11816; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end1817; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]1818; GFX6-NEXT: s_waitcnt expcnt(0)1819; GFX6-NEXT: s_setpc_b64 s[30:31]1820 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 2561821 %result = atomicrmw fadd ptr addrspace(7) %gep, float %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0, !amdgpu.ignore.denormal.mode !01822 ret float %result1823}1824 1825; --------------------------------------------------------------------1826; double1827; --------------------------------------------------------------------1828 1829define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {1830; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:1831; GFX12: ; %bb.0:1832; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01833; GFX12-NEXT: s_wait_expcnt 0x01834; GFX12-NEXT: s_wait_samplecnt 0x01835; GFX12-NEXT: s_wait_bvhcnt 0x01836; GFX12-NEXT: s_wait_kmcnt 0x01837; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v01838; GFX12-NEXT: v_mov_b32_e32 v0, s161839; GFX12-NEXT: v_mov_b32_e32 v10, s161840; GFX12-NEXT: s_mov_b32 s4, 01841; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:20481842; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start1843; GFX12-NEXT: ; =>This Inner Loop Header: Depth=11844; GFX12-NEXT: s_wait_loadcnt 0x01845; GFX12-NEXT: v_add_f64_e32 v[6:7], v[8:9], v[4:5]1846; GFX12-NEXT: s_wait_storecnt 0x01847; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)1848; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v71849; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v91850; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN1851; GFX12-NEXT: s_wait_loadcnt 0x01852; GFX12-NEXT: global_inv scope:SCOPE_DEV1853; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]1854; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v01855; GFX12-NEXT: s_or_b32 s4, vcc_lo, s41856; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1857; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s41858; GFX12-NEXT: s_cbranch_execnz .LBB8_11859; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end1860; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s41861; GFX12-NEXT: s_wait_loadcnt 0x01862; GFX12-NEXT: s_setpc_b64 s[30:31]1863;1864; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:1865; GFX942: ; %bb.0:1866; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1867; GFX942-NEXT: v_mov_b32_e32 v2, s161868; GFX942-NEXT: buffer_wbl2 sc11869; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc01870; GFX942-NEXT: s_waitcnt vmcnt(0)1871; GFX942-NEXT: buffer_inv sc11872; GFX942-NEXT: s_setpc_b64 s[30:31]1873;1874; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:1875; GFX11: ; %bb.0:1876; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1877; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v01878; GFX11-NEXT: v_mov_b32_e32 v0, s161879; GFX11-NEXT: v_mov_b32_e32 v10, s161880; GFX11-NEXT: s_mov_b32 s4, 01881; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:20481882; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start1883; GFX11-NEXT: ; =>This Inner Loop Header: Depth=11884; GFX11-NEXT: s_waitcnt vmcnt(0)1885; GFX11-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]1886; GFX11-NEXT: s_waitcnt_vscnt null, 0x01887; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)1888; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v71889; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v91890; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc1891; GFX11-NEXT: s_waitcnt vmcnt(0)1892; GFX11-NEXT: buffer_gl1_inv1893; GFX11-NEXT: buffer_gl0_inv1894; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]1895; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v01896; GFX11-NEXT: s_or_b32 s4, vcc_lo, s41897; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1898; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s41899; GFX11-NEXT: s_cbranch_execnz .LBB8_11900; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end1901; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s41902; GFX11-NEXT: s_setpc_b64 s[30:31]1903;1904; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:1905; GFX10: ; %bb.0:1906; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1907; GFX10-NEXT: v_mov_b32_e32 v4, v01908; GFX10-NEXT: v_mov_b32_e32 v0, s201909; GFX10-NEXT: v_mov_b32_e32 v5, v11910; GFX10-NEXT: v_mov_b32_e32 v10, s201911; GFX10-NEXT: s_mov_b32 s4, 01912; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20481913; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start1914; GFX10-NEXT: ; =>This Inner Loop Header: Depth=11915; GFX10-NEXT: s_waitcnt vmcnt(0)1916; GFX10-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]1917; GFX10-NEXT: s_waitcnt_vscnt null, 0x01918; GFX10-NEXT: v_mov_b32_e32 v0, v61919; GFX10-NEXT: v_mov_b32_e32 v1, v71920; GFX10-NEXT: v_mov_b32_e32 v2, v81921; GFX10-NEXT: v_mov_b32_e32 v3, v91922; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc1923; GFX10-NEXT: s_waitcnt vmcnt(0)1924; GFX10-NEXT: buffer_gl1_inv1925; GFX10-NEXT: buffer_gl0_inv1926; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]1927; GFX10-NEXT: v_mov_b32_e32 v9, v11928; GFX10-NEXT: v_mov_b32_e32 v8, v01929; GFX10-NEXT: s_or_b32 s4, vcc_lo, s41930; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s41931; GFX10-NEXT: s_cbranch_execnz .LBB8_11932; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end1933; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s41934; GFX10-NEXT: s_setpc_b64 s[30:31]1935;1936; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:1937; GFX90A: ; %bb.0:1938; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1939; GFX90A-NEXT: v_mov_b32_e32 v2, s201940; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[16:19], 0 offen offset:2048 glc1941; GFX90A-NEXT: s_waitcnt vmcnt(0)1942; GFX90A-NEXT: buffer_wbinvl11943; GFX90A-NEXT: s_setpc_b64 s[30:31]1944;1945; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:1946; GFX908: ; %bb.0:1947; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1948; GFX908-NEXT: v_mov_b32_e32 v4, v01949; GFX908-NEXT: v_mov_b32_e32 v0, s201950; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20481951; GFX908-NEXT: v_mov_b32_e32 v5, v11952; GFX908-NEXT: s_mov_b64 s[4:5], 01953; GFX908-NEXT: v_mov_b32_e32 v10, s201954; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start1955; GFX908-NEXT: ; =>This Inner Loop Header: Depth=11956; GFX908-NEXT: s_waitcnt vmcnt(0)1957; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]1958; GFX908-NEXT: v_mov_b32_e32 v0, v61959; GFX908-NEXT: v_mov_b32_e32 v1, v71960; GFX908-NEXT: v_mov_b32_e32 v2, v81961; GFX908-NEXT: v_mov_b32_e32 v3, v91962; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc1963; GFX908-NEXT: s_waitcnt vmcnt(0)1964; GFX908-NEXT: buffer_wbinvl11965; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]1966; GFX908-NEXT: v_mov_b32_e32 v9, v11967; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1968; GFX908-NEXT: v_mov_b32_e32 v8, v01969; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]1970; GFX908-NEXT: s_cbranch_execnz .LBB8_11971; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end1972; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]1973; GFX908-NEXT: s_setpc_b64 s[30:31]1974;1975; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:1976; GFX8: ; %bb.0:1977; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1978; GFX8-NEXT: v_mov_b32_e32 v4, v01979; GFX8-NEXT: v_mov_b32_e32 v0, s201980; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20481981; GFX8-NEXT: v_mov_b32_e32 v5, v11982; GFX8-NEXT: s_mov_b64 s[4:5], 01983; GFX8-NEXT: v_mov_b32_e32 v10, s201984; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start1985; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11986; GFX8-NEXT: s_waitcnt vmcnt(0)1987; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]1988; GFX8-NEXT: v_mov_b32_e32 v0, v61989; GFX8-NEXT: v_mov_b32_e32 v1, v71990; GFX8-NEXT: v_mov_b32_e32 v2, v81991; GFX8-NEXT: v_mov_b32_e32 v3, v91992; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc1993; GFX8-NEXT: s_waitcnt vmcnt(0)1994; GFX8-NEXT: buffer_wbinvl11995; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]1996; GFX8-NEXT: v_mov_b32_e32 v9, v11997; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1998; GFX8-NEXT: v_mov_b32_e32 v8, v01999; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]2000; GFX8-NEXT: s_cbranch_execnz .LBB8_12001; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2002; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]2003; GFX8-NEXT: s_setpc_b64 s[30:31]2004;2005; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:2006; GFX7: ; %bb.0:2007; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2008; GFX7-NEXT: v_mov_b32_e32 v4, v02009; GFX7-NEXT: v_mov_b32_e32 v0, s202010; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20482011; GFX7-NEXT: v_mov_b32_e32 v5, v12012; GFX7-NEXT: s_mov_b64 s[4:5], 02013; GFX7-NEXT: v_mov_b32_e32 v10, s202014; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start2015; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12016; GFX7-NEXT: s_waitcnt vmcnt(0)2017; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]2018; GFX7-NEXT: v_mov_b32_e32 v0, v62019; GFX7-NEXT: v_mov_b32_e32 v1, v72020; GFX7-NEXT: v_mov_b32_e32 v2, v82021; GFX7-NEXT: v_mov_b32_e32 v3, v92022; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc2023; GFX7-NEXT: s_waitcnt vmcnt(0)2024; GFX7-NEXT: buffer_wbinvl12025; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2026; GFX7-NEXT: v_mov_b32_e32 v9, v12027; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2028; GFX7-NEXT: v_mov_b32_e32 v8, v02029; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]2030; GFX7-NEXT: s_cbranch_execnz .LBB8_12031; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2032; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2033; GFX7-NEXT: s_setpc_b64 s[30:31]2034;2035; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory:2036; GFX6: ; %bb.0:2037; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2038; GFX6-NEXT: v_mov_b32_e32 v4, v02039; GFX6-NEXT: v_mov_b32_e32 v0, s202040; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20482041; GFX6-NEXT: s_add_i32 s6, s20, 0x8002042; GFX6-NEXT: v_mov_b32_e32 v5, v12043; GFX6-NEXT: s_mov_b64 s[4:5], 02044; GFX6-NEXT: v_mov_b32_e32 v10, s62045; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start2046; GFX6-NEXT: ; =>This Inner Loop Header: Depth=12047; GFX6-NEXT: s_waitcnt vmcnt(0)2048; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]2049; GFX6-NEXT: s_waitcnt expcnt(0)2050; GFX6-NEXT: v_mov_b32_e32 v0, v62051; GFX6-NEXT: v_mov_b32_e32 v1, v72052; GFX6-NEXT: v_mov_b32_e32 v2, v82053; GFX6-NEXT: v_mov_b32_e32 v3, v92054; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc2055; GFX6-NEXT: s_waitcnt vmcnt(0)2056; GFX6-NEXT: buffer_wbinvl12057; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2058; GFX6-NEXT: v_mov_b32_e32 v9, v12059; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2060; GFX6-NEXT: v_mov_b32_e32 v8, v02061; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]2062; GFX6-NEXT: s_cbranch_execnz .LBB8_12063; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end2064; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]2065; GFX6-NEXT: s_waitcnt expcnt(0)2066; GFX6-NEXT: s_setpc_b64 s[30:31]2067 %gep = getelementptr double, ptr addrspace(7) %ptr, i32 2562068 %result = atomicrmw fadd ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02069 ret double %result2070}2071 2072define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {2073; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:2074; GFX12: ; %bb.0:2075; GFX12-NEXT: s_wait_loadcnt_dscnt 0x02076; GFX12-NEXT: s_wait_expcnt 0x02077; GFX12-NEXT: s_wait_samplecnt 0x02078; GFX12-NEXT: s_wait_bvhcnt 0x02079; GFX12-NEXT: s_wait_kmcnt 0x02080; GFX12-NEXT: v_mov_b32_e32 v2, s162081; GFX12-NEXT: v_mov_b32_e32 v6, s162082; GFX12-NEXT: s_mov_b32 s4, 02083; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:20482084; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start2085; GFX12-NEXT: ; =>This Inner Loop Header: Depth=12086; GFX12-NEXT: s_wait_loadcnt 0x02087; GFX12-NEXT: v_add_f64_e32 v[2:3], v[4:5], v[0:1]2088; GFX12-NEXT: v_dual_mov_b32 v10, v5 :: v_dual_mov_b32 v9, v42089; GFX12-NEXT: s_wait_storecnt 0x02090; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)2091; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v22092; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN2093; GFX12-NEXT: s_wait_loadcnt 0x02094; GFX12-NEXT: global_inv scope:SCOPE_DEV2095; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[4:5]2096; GFX12-NEXT: v_dual_mov_b32 v4, v7 :: v_dual_mov_b32 v5, v82097; GFX12-NEXT: s_or_b32 s4, vcc_lo, s42098; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2099; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s42100; GFX12-NEXT: s_cbranch_execnz .LBB9_12101; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end2102; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s42103; GFX12-NEXT: s_wait_loadcnt 0x02104; GFX12-NEXT: s_setpc_b64 s[30:31]2105;2106; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:2107; GFX942: ; %bb.0:2108; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2109; GFX942-NEXT: v_mov_b32_e32 v2, s162110; GFX942-NEXT: buffer_wbl2 sc12111; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen offset:20482112; GFX942-NEXT: s_waitcnt vmcnt(0)2113; GFX942-NEXT: buffer_inv sc12114; GFX942-NEXT: s_setpc_b64 s[30:31]2115;2116; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:2117; GFX11: ; %bb.0:2118; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2119; GFX11-NEXT: v_mov_b32_e32 v2, s162120; GFX11-NEXT: v_mov_b32_e32 v6, s162121; GFX11-NEXT: s_mov_b32 s4, 02122; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:20482123; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start2124; GFX11-NEXT: ; =>This Inner Loop Header: Depth=12125; GFX11-NEXT: s_waitcnt vmcnt(0)2126; GFX11-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]2127; GFX11-NEXT: v_dual_mov_b32 v10, v5 :: v_dual_mov_b32 v9, v42128; GFX11-NEXT: s_waitcnt_vscnt null, 0x02129; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)2130; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v22131; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen offset:2048 glc2132; GFX11-NEXT: s_waitcnt vmcnt(0)2133; GFX11-NEXT: buffer_gl1_inv2134; GFX11-NEXT: buffer_gl0_inv2135; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[4:5]2136; GFX11-NEXT: v_dual_mov_b32 v4, v7 :: v_dual_mov_b32 v5, v82137; GFX11-NEXT: s_or_b32 s4, vcc_lo, s42138; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2139; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s42140; GFX11-NEXT: s_cbranch_execnz .LBB9_12141; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end2142; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s42143; GFX11-NEXT: s_setpc_b64 s[30:31]2144;2145; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:2146; GFX10: ; %bb.0:2147; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2148; GFX10-NEXT: v_mov_b32_e32 v2, s202149; GFX10-NEXT: v_mov_b32_e32 v6, s202150; GFX10-NEXT: s_mov_b32 s4, 02151; GFX10-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:20482152; GFX10-NEXT: .LBB9_1: ; %atomicrmw.start2153; GFX10-NEXT: ; =>This Inner Loop Header: Depth=12154; GFX10-NEXT: s_waitcnt vmcnt(0)2155; GFX10-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]2156; GFX10-NEXT: v_mov_b32_e32 v10, v52157; GFX10-NEXT: v_mov_b32_e32 v9, v42158; GFX10-NEXT: s_waitcnt_vscnt null, 0x02159; GFX10-NEXT: v_mov_b32_e32 v8, v32160; GFX10-NEXT: v_mov_b32_e32 v7, v22161; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc2162; GFX10-NEXT: s_waitcnt vmcnt(0)2163; GFX10-NEXT: buffer_gl1_inv2164; GFX10-NEXT: buffer_gl0_inv2165; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[4:5]2166; GFX10-NEXT: v_mov_b32_e32 v4, v72167; GFX10-NEXT: v_mov_b32_e32 v5, v82168; GFX10-NEXT: s_or_b32 s4, vcc_lo, s42169; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s42170; GFX10-NEXT: s_cbranch_execnz .LBB9_12171; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end2172; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s42173; GFX10-NEXT: s_setpc_b64 s[30:31]2174;2175; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:2176; GFX90A: ; %bb.0:2177; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2178; GFX90A-NEXT: v_mov_b32_e32 v2, s202179; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[16:19], 0 offen offset:20482180; GFX90A-NEXT: s_waitcnt vmcnt(0)2181; GFX90A-NEXT: buffer_wbinvl12182; GFX90A-NEXT: s_setpc_b64 s[30:31]2183;2184; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:2185; GFX908: ; %bb.0:2186; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2187; GFX908-NEXT: v_mov_b32_e32 v2, s202188; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:20482189; GFX908-NEXT: s_mov_b64 s[4:5], 02190; GFX908-NEXT: v_mov_b32_e32 v6, s202191; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start2192; GFX908-NEXT: ; =>This Inner Loop Header: Depth=12193; GFX908-NEXT: s_waitcnt vmcnt(0)2194; GFX908-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]2195; GFX908-NEXT: v_mov_b32_e32 v10, v52196; GFX908-NEXT: v_mov_b32_e32 v9, v42197; GFX908-NEXT: v_mov_b32_e32 v8, v32198; GFX908-NEXT: v_mov_b32_e32 v7, v22199; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc2200; GFX908-NEXT: s_waitcnt vmcnt(0)2201; GFX908-NEXT: buffer_wbinvl12202; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[4:5]2203; GFX908-NEXT: v_mov_b32_e32 v4, v72204; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2205; GFX908-NEXT: v_mov_b32_e32 v5, v82206; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]2207; GFX908-NEXT: s_cbranch_execnz .LBB9_12208; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end2209; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]2210; GFX908-NEXT: s_setpc_b64 s[30:31]2211;2212; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:2213; GFX8: ; %bb.0:2214; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2215; GFX8-NEXT: v_mov_b32_e32 v2, s202216; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:20482217; GFX8-NEXT: s_mov_b64 s[4:5], 02218; GFX8-NEXT: v_mov_b32_e32 v6, s202219; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start2220; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12221; GFX8-NEXT: s_waitcnt vmcnt(0)2222; GFX8-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]2223; GFX8-NEXT: v_mov_b32_e32 v10, v52224; GFX8-NEXT: v_mov_b32_e32 v9, v42225; GFX8-NEXT: v_mov_b32_e32 v8, v32226; GFX8-NEXT: v_mov_b32_e32 v7, v22227; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc2228; GFX8-NEXT: s_waitcnt vmcnt(0)2229; GFX8-NEXT: buffer_wbinvl12230; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[4:5]2231; GFX8-NEXT: v_mov_b32_e32 v4, v72232; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2233; GFX8-NEXT: v_mov_b32_e32 v5, v82234; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]2235; GFX8-NEXT: s_cbranch_execnz .LBB9_12236; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2237; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]2238; GFX8-NEXT: s_setpc_b64 s[30:31]2239;2240; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:2241; GFX7: ; %bb.0:2242; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2243; GFX7-NEXT: v_mov_b32_e32 v2, s202244; GFX7-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:20482245; GFX7-NEXT: s_mov_b64 s[4:5], 02246; GFX7-NEXT: v_mov_b32_e32 v6, s202247; GFX7-NEXT: .LBB9_1: ; %atomicrmw.start2248; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12249; GFX7-NEXT: s_waitcnt vmcnt(0)2250; GFX7-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]2251; GFX7-NEXT: v_mov_b32_e32 v10, v52252; GFX7-NEXT: v_mov_b32_e32 v9, v42253; GFX7-NEXT: v_mov_b32_e32 v8, v32254; GFX7-NEXT: v_mov_b32_e32 v7, v22255; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc2256; GFX7-NEXT: s_waitcnt vmcnt(0)2257; GFX7-NEXT: buffer_wbinvl12258; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[4:5]2259; GFX7-NEXT: v_mov_b32_e32 v4, v72260; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2261; GFX7-NEXT: v_mov_b32_e32 v5, v82262; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]2263; GFX7-NEXT: s_cbranch_execnz .LBB9_12264; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2265; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2266; GFX7-NEXT: s_setpc_b64 s[30:31]2267;2268; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_grained_memory:2269; GFX6: ; %bb.0:2270; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2271; GFX6-NEXT: v_mov_b32_e32 v2, s202272; GFX6-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:20482273; GFX6-NEXT: s_add_i32 s6, s20, 0x8002274; GFX6-NEXT: s_mov_b64 s[4:5], 02275; GFX6-NEXT: v_mov_b32_e32 v6, s62276; GFX6-NEXT: .LBB9_1: ; %atomicrmw.start2277; GFX6-NEXT: ; =>This Inner Loop Header: Depth=12278; GFX6-NEXT: s_waitcnt vmcnt(0)2279; GFX6-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]2280; GFX6-NEXT: s_waitcnt expcnt(0)2281; GFX6-NEXT: v_mov_b32_e32 v10, v52282; GFX6-NEXT: v_mov_b32_e32 v9, v42283; GFX6-NEXT: v_mov_b32_e32 v8, v32284; GFX6-NEXT: v_mov_b32_e32 v7, v22285; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen glc2286; GFX6-NEXT: s_waitcnt vmcnt(0)2287; GFX6-NEXT: buffer_wbinvl12288; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[4:5]2289; GFX6-NEXT: v_mov_b32_e32 v4, v72290; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2291; GFX6-NEXT: v_mov_b32_e32 v5, v82292; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]2293; GFX6-NEXT: s_cbranch_execnz .LBB9_12294; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end2295; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]2296; GFX6-NEXT: s_waitcnt expcnt(0)2297; GFX6-NEXT: s_setpc_b64 s[30:31]2298 %gep = getelementptr double, ptr addrspace(7) %ptr, i32 2562299 %unused = atomicrmw fadd ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02300 ret void2301}2302 2303define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, double %val) #0 {2304; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:2305; GFX12: ; %bb.0:2306; GFX12-NEXT: s_wait_loadcnt_dscnt 0x02307; GFX12-NEXT: s_wait_expcnt 0x02308; GFX12-NEXT: s_wait_samplecnt 0x02309; GFX12-NEXT: s_wait_bvhcnt 0x02310; GFX12-NEXT: s_wait_kmcnt 0x02311; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v22312; GFX12-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v02313; GFX12-NEXT: s_mov_b32 s1, exec_lo2314; GFX12-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=12315; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)2316; GFX12-NEXT: v_readfirstlane_b32 s4, v92317; GFX12-NEXT: v_readfirstlane_b32 s5, v102318; GFX12-NEXT: v_readfirstlane_b32 s6, v72319; GFX12-NEXT: v_readfirstlane_b32 s7, v82320; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)2321; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)2322; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]2323; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]2324; GFX12-NEXT: s_and_b32 s0, vcc_lo, s02325; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2326; GFX12-NEXT: s_and_saveexec_b32 s0, s02327; GFX12-NEXT: s_wait_loadcnt 0x02328; GFX12-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], null offen offset:20482329; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s02330; GFX12-NEXT: s_cbranch_execnz .LBB10_12331; GFX12-NEXT: ; %bb.2:2332; GFX12-NEXT: s_mov_b32 exec_lo, s12333; GFX12-NEXT: s_mov_b32 s1, 02334; GFX12-NEXT: .LBB10_3: ; %atomicrmw.start2335; GFX12-NEXT: ; =>This Loop Header: Depth=12336; GFX12-NEXT: ; Child Loop BB10_4 Depth 22337; GFX12-NEXT: s_wait_loadcnt 0x02338; GFX12-NEXT: v_add_f64_e32 v[11:12], v[13:14], v[5:6]2339; GFX12-NEXT: s_mov_b32 s2, exec_lo2340; GFX12-NEXT: s_wait_storecnt 0x02341; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)2342; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v122343; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v142344; GFX12-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=12345; GFX12-NEXT: ; => This Inner Loop Header: Depth=22346; GFX12-NEXT: v_readfirstlane_b32 s4, v92347; GFX12-NEXT: v_readfirstlane_b32 s5, v102348; GFX12-NEXT: v_readfirstlane_b32 s6, v72349; GFX12-NEXT: v_readfirstlane_b32 s7, v82350; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)2351; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)2352; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]2353; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]2354; GFX12-NEXT: s_and_b32 s0, vcc_lo, s02355; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2356; GFX12-NEXT: s_and_saveexec_b32 s0, s02357; GFX12-NEXT: s_wait_loadcnt 0x02358; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], null offen offset:2048 th:TH_ATOMIC_RETURN2359; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s02360; GFX12-NEXT: s_cbranch_execnz .LBB10_42361; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=12362; GFX12-NEXT: s_mov_b32 exec_lo, s22363; GFX12-NEXT: s_wait_loadcnt 0x02364; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]2365; GFX12-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v02366; GFX12-NEXT: global_inv scope:SCOPE_DEV2367; GFX12-NEXT: s_or_b32 s1, vcc_lo, s12368; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2369; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s12370; GFX12-NEXT: s_cbranch_execnz .LBB10_32371; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end2372; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s12373; GFX12-NEXT: s_wait_loadcnt 0x02374; GFX12-NEXT: s_setpc_b64 s[30:31]2375;2376; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:2377; GFX942: ; %bb.0:2378; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2379; GFX942-NEXT: v_mov_b32_e32 v7, v62380; GFX942-NEXT: v_mov_b32_e32 v6, v52381; GFX942-NEXT: s_mov_b64 s[2:3], exec2382; GFX942-NEXT: buffer_wbl2 sc12383; GFX942-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=12384; GFX942-NEXT: v_readfirstlane_b32 s4, v02385; GFX942-NEXT: v_readfirstlane_b32 s5, v12386; GFX942-NEXT: v_readfirstlane_b32 s6, v22387; GFX942-NEXT: v_readfirstlane_b32 s7, v32388; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]2389; GFX942-NEXT: s_nop 02390; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]2391; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]2392; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]2393; GFX942-NEXT: s_waitcnt vmcnt(0)2394; GFX942-NEXT: buffer_atomic_add_f64 v[6:7], v4, s[4:7], 0 offen offset:2048 sc02395; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr32396; GFX942-NEXT: ; implicit-def: $vgpr42397; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]2398; GFX942-NEXT: s_cbranch_execnz .LBB10_12399; GFX942-NEXT: ; %bb.2:2400; GFX942-NEXT: s_mov_b64 exec, s[2:3]2401; GFX942-NEXT: s_waitcnt vmcnt(0)2402; GFX942-NEXT: v_mov_b32_e32 v0, v62403; GFX942-NEXT: v_mov_b32_e32 v1, v72404; GFX942-NEXT: buffer_inv sc12405; GFX942-NEXT: s_setpc_b64 s[30:31]2406;2407; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:2408; GFX11: ; %bb.0:2409; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2410; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v22411; GFX11-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v02412; GFX11-NEXT: s_mov_b32 s1, 02413; GFX11-NEXT: s_mov_b32 s2, exec_lo2414; GFX11-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=12415; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)2416; GFX11-NEXT: v_readfirstlane_b32 s4, v92417; GFX11-NEXT: v_readfirstlane_b32 s5, v102418; GFX11-NEXT: v_readfirstlane_b32 s6, v72419; GFX11-NEXT: v_readfirstlane_b32 s7, v82420; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)2421; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]2422; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]2423; GFX11-NEXT: s_and_b32 s0, vcc_lo, s02424; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2425; GFX11-NEXT: s_and_saveexec_b32 s0, s02426; GFX11-NEXT: buffer_load_b64 v[13:14], v4, s[4:7], 0 offen offset:20482427; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s02428; GFX11-NEXT: s_cbranch_execnz .LBB10_12429; GFX11-NEXT: ; %bb.2:2430; GFX11-NEXT: s_mov_b32 exec_lo, s22431; GFX11-NEXT: .p2align 62432; GFX11-NEXT: .LBB10_3: ; %atomicrmw.start2433; GFX11-NEXT: ; =>This Loop Header: Depth=12434; GFX11-NEXT: ; Child Loop BB10_4 Depth 22435; GFX11-NEXT: s_waitcnt vmcnt(0)2436; GFX11-NEXT: v_add_f64 v[11:12], v[13:14], v[5:6]2437; GFX11-NEXT: s_mov_b32 s2, exec_lo2438; GFX11-NEXT: s_waitcnt_vscnt null, 0x02439; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)2440; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v122441; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v142442; GFX11-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=12443; GFX11-NEXT: ; => This Inner Loop Header: Depth=22444; GFX11-NEXT: v_readfirstlane_b32 s4, v92445; GFX11-NEXT: v_readfirstlane_b32 s5, v102446; GFX11-NEXT: v_readfirstlane_b32 s6, v72447; GFX11-NEXT: v_readfirstlane_b32 s7, v82448; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)2449; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]2450; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[7:8]2451; GFX11-NEXT: s_and_b32 s0, vcc_lo, s02452; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2453; GFX11-NEXT: s_and_saveexec_b32 s0, s02454; GFX11-NEXT: s_waitcnt vmcnt(0)2455; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v4, s[4:7], 0 offen offset:2048 glc2456; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s02457; GFX11-NEXT: s_cbranch_execnz .LBB10_42458; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=12459; GFX11-NEXT: s_mov_b32 exec_lo, s22460; GFX11-NEXT: s_waitcnt vmcnt(0)2461; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]2462; GFX11-NEXT: v_dual_mov_b32 v14, v1 :: v_dual_mov_b32 v13, v02463; GFX11-NEXT: buffer_gl1_inv2464; GFX11-NEXT: buffer_gl0_inv2465; GFX11-NEXT: s_or_b32 s1, vcc_lo, s12466; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2467; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s12468; GFX11-NEXT: s_cbranch_execnz .LBB10_32469; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end2470; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s12471; GFX11-NEXT: s_setpc_b64 s[30:31]2472;2473; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:2474; GFX10: ; %bb.0:2475; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2476; GFX10-NEXT: v_mov_b32_e32 v8, v32477; GFX10-NEXT: v_mov_b32_e32 v7, v22478; GFX10-NEXT: v_mov_b32_e32 v10, v12479; GFX10-NEXT: v_mov_b32_e32 v9, v02480; GFX10-NEXT: s_mov_b32 s5, 02481; GFX10-NEXT: s_mov_b32 s6, exec_lo2482; GFX10-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=12483; GFX10-NEXT: v_readfirstlane_b32 s8, v92484; GFX10-NEXT: v_readfirstlane_b32 s9, v102485; GFX10-NEXT: v_readfirstlane_b32 s10, v72486; GFX10-NEXT: v_readfirstlane_b32 s11, v82487; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[9:10]2488; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[7:8]2489; GFX10-NEXT: s_and_b32 s4, vcc_lo, s42490; GFX10-NEXT: s_and_saveexec_b32 s4, s42491; GFX10-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:20482492; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2493; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s42494; GFX10-NEXT: s_cbranch_execnz .LBB10_12495; GFX10-NEXT: ; %bb.2:2496; GFX10-NEXT: s_mov_b32 exec_lo, s62497; GFX10-NEXT: .LBB10_3: ; %atomicrmw.start2498; GFX10-NEXT: ; =>This Loop Header: Depth=12499; GFX10-NEXT: ; Child Loop BB10_4 Depth 22500; GFX10-NEXT: s_waitcnt vmcnt(0)2501; GFX10-NEXT: v_add_f64 v[11:12], v[13:14], v[5:6]2502; GFX10-NEXT: s_mov_b32 s6, exec_lo2503; GFX10-NEXT: s_waitcnt_vscnt null, 0x02504; GFX10-NEXT: v_mov_b32_e32 v0, v112505; GFX10-NEXT: v_mov_b32_e32 v1, v122506; GFX10-NEXT: v_mov_b32_e32 v2, v132507; GFX10-NEXT: v_mov_b32_e32 v3, v142508; GFX10-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=12509; GFX10-NEXT: ; => This Inner Loop Header: Depth=22510; GFX10-NEXT: v_readfirstlane_b32 s8, v92511; GFX10-NEXT: v_readfirstlane_b32 s9, v102512; GFX10-NEXT: v_readfirstlane_b32 s10, v72513; GFX10-NEXT: v_readfirstlane_b32 s11, v82514; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[9:10]2515; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[7:8]2516; GFX10-NEXT: s_and_b32 s4, vcc_lo, s42517; GFX10-NEXT: s_and_saveexec_b32 s4, s42518; GFX10-NEXT: s_waitcnt vmcnt(0)2519; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc2520; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2521; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s42522; GFX10-NEXT: s_cbranch_execnz .LBB10_42523; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=12524; GFX10-NEXT: s_mov_b32 exec_lo, s62525; GFX10-NEXT: s_waitcnt vmcnt(0)2526; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[13:14]2527; GFX10-NEXT: v_mov_b32_e32 v14, v12528; GFX10-NEXT: v_mov_b32_e32 v13, v02529; GFX10-NEXT: buffer_gl1_inv2530; GFX10-NEXT: buffer_gl0_inv2531; GFX10-NEXT: s_or_b32 s5, vcc_lo, s52532; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)2533; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s52534; GFX10-NEXT: s_cbranch_execnz .LBB10_32535; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end2536; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s52537; GFX10-NEXT: s_setpc_b64 s[30:31]2538;2539; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:2540; GFX90A: ; %bb.0:2541; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2542; GFX90A-NEXT: v_mov_b32_e32 v7, v62543; GFX90A-NEXT: v_mov_b32_e32 v6, v52544; GFX90A-NEXT: s_mov_b64 s[6:7], exec2545; GFX90A-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=12546; GFX90A-NEXT: v_readfirstlane_b32 s8, v02547; GFX90A-NEXT: v_readfirstlane_b32 s9, v12548; GFX90A-NEXT: v_readfirstlane_b32 s10, v22549; GFX90A-NEXT: v_readfirstlane_b32 s11, v32550; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]2551; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]2552; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]2553; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]2554; GFX90A-NEXT: s_waitcnt vmcnt(0)2555; GFX90A-NEXT: buffer_atomic_add_f64 v[6:7], v4, s[8:11], 0 offen offset:2048 glc2556; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr32557; GFX90A-NEXT: ; implicit-def: $vgpr42558; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]2559; GFX90A-NEXT: s_cbranch_execnz .LBB10_12560; GFX90A-NEXT: ; %bb.2:2561; GFX90A-NEXT: s_mov_b64 exec, s[6:7]2562; GFX90A-NEXT: s_waitcnt vmcnt(0)2563; GFX90A-NEXT: v_mov_b32_e32 v0, v62564; GFX90A-NEXT: v_mov_b32_e32 v1, v72565; GFX90A-NEXT: buffer_wbinvl12566; GFX90A-NEXT: s_setpc_b64 s[30:31]2567;2568; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:2569; GFX908: ; %bb.0:2570; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2571; GFX908-NEXT: v_mov_b32_e32 v8, v32572; GFX908-NEXT: v_mov_b32_e32 v7, v22573; GFX908-NEXT: v_mov_b32_e32 v10, v12574; GFX908-NEXT: v_mov_b32_e32 v9, v02575; GFX908-NEXT: s_mov_b64 s[6:7], exec2576; GFX908-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=12577; GFX908-NEXT: v_readfirstlane_b32 s8, v92578; GFX908-NEXT: v_readfirstlane_b32 s9, v102579; GFX908-NEXT: v_readfirstlane_b32 s10, v72580; GFX908-NEXT: v_readfirstlane_b32 s11, v82581; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]2582; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]2583; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]2584; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]2585; GFX908-NEXT: s_nop 02586; GFX908-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:20482587; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]2588; GFX908-NEXT: s_cbranch_execnz .LBB10_12589; GFX908-NEXT: ; %bb.2:2590; GFX908-NEXT: s_mov_b64 exec, s[6:7]2591; GFX908-NEXT: s_mov_b64 s[6:7], 02592; GFX908-NEXT: .LBB10_3: ; %atomicrmw.start2593; GFX908-NEXT: ; =>This Loop Header: Depth=12594; GFX908-NEXT: ; Child Loop BB10_4 Depth 22595; GFX908-NEXT: s_waitcnt vmcnt(0)2596; GFX908-NEXT: v_add_f64 v[11:12], v[13:14], v[5:6]2597; GFX908-NEXT: s_mov_b64 s[12:13], exec2598; GFX908-NEXT: v_mov_b32_e32 v0, v112599; GFX908-NEXT: v_mov_b32_e32 v1, v122600; GFX908-NEXT: v_mov_b32_e32 v2, v132601; GFX908-NEXT: v_mov_b32_e32 v3, v142602; GFX908-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=12603; GFX908-NEXT: ; => This Inner Loop Header: Depth=22604; GFX908-NEXT: v_readfirstlane_b32 s8, v92605; GFX908-NEXT: v_readfirstlane_b32 s9, v102606; GFX908-NEXT: v_readfirstlane_b32 s10, v72607; GFX908-NEXT: v_readfirstlane_b32 s11, v82608; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]2609; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]2610; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]2611; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]2612; GFX908-NEXT: s_waitcnt vmcnt(0)2613; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc2614; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]2615; GFX908-NEXT: s_cbranch_execnz .LBB10_42616; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=12617; GFX908-NEXT: s_mov_b64 exec, s[12:13]2618; GFX908-NEXT: s_waitcnt vmcnt(0)2619; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]2620; GFX908-NEXT: v_mov_b32_e32 v14, v12621; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]2622; GFX908-NEXT: v_mov_b32_e32 v13, v02623; GFX908-NEXT: buffer_wbinvl12624; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]2625; GFX908-NEXT: s_cbranch_execnz .LBB10_32626; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end2627; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]2628; GFX908-NEXT: s_setpc_b64 s[30:31]2629;2630; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:2631; GFX8: ; %bb.0:2632; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2633; GFX8-NEXT: v_mov_b32_e32 v8, v32634; GFX8-NEXT: v_mov_b32_e32 v7, v22635; GFX8-NEXT: v_mov_b32_e32 v10, v12636; GFX8-NEXT: v_mov_b32_e32 v9, v02637; GFX8-NEXT: s_mov_b64 s[6:7], exec2638; GFX8-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=12639; GFX8-NEXT: v_readfirstlane_b32 s8, v92640; GFX8-NEXT: v_readfirstlane_b32 s9, v102641; GFX8-NEXT: v_readfirstlane_b32 s10, v72642; GFX8-NEXT: v_readfirstlane_b32 s11, v82643; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]2644; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]2645; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]2646; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]2647; GFX8-NEXT: s_nop 02648; GFX8-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:20482649; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]2650; GFX8-NEXT: s_cbranch_execnz .LBB10_12651; GFX8-NEXT: ; %bb.2:2652; GFX8-NEXT: s_mov_b64 exec, s[6:7]2653; GFX8-NEXT: s_mov_b64 s[6:7], 02654; GFX8-NEXT: .LBB10_3: ; %atomicrmw.start2655; GFX8-NEXT: ; =>This Loop Header: Depth=12656; GFX8-NEXT: ; Child Loop BB10_4 Depth 22657; GFX8-NEXT: s_waitcnt vmcnt(0)2658; GFX8-NEXT: v_add_f64 v[11:12], v[13:14], v[5:6]2659; GFX8-NEXT: s_mov_b64 s[12:13], exec2660; GFX8-NEXT: v_mov_b32_e32 v0, v112661; GFX8-NEXT: v_mov_b32_e32 v1, v122662; GFX8-NEXT: v_mov_b32_e32 v2, v132663; GFX8-NEXT: v_mov_b32_e32 v3, v142664; GFX8-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=12665; GFX8-NEXT: ; => This Inner Loop Header: Depth=22666; GFX8-NEXT: v_readfirstlane_b32 s8, v92667; GFX8-NEXT: v_readfirstlane_b32 s9, v102668; GFX8-NEXT: v_readfirstlane_b32 s10, v72669; GFX8-NEXT: v_readfirstlane_b32 s11, v82670; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]2671; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]2672; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]2673; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]2674; GFX8-NEXT: s_waitcnt vmcnt(0)2675; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc2676; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]2677; GFX8-NEXT: s_cbranch_execnz .LBB10_42678; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=12679; GFX8-NEXT: s_mov_b64 exec, s[12:13]2680; GFX8-NEXT: s_waitcnt vmcnt(0)2681; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]2682; GFX8-NEXT: v_mov_b32_e32 v14, v12683; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]2684; GFX8-NEXT: v_mov_b32_e32 v13, v02685; GFX8-NEXT: buffer_wbinvl12686; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]2687; GFX8-NEXT: s_cbranch_execnz .LBB10_32688; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end2689; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]2690; GFX8-NEXT: s_setpc_b64 s[30:31]2691;2692; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:2693; GFX7: ; %bb.0:2694; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2695; GFX7-NEXT: v_mov_b32_e32 v8, v32696; GFX7-NEXT: v_mov_b32_e32 v7, v22697; GFX7-NEXT: v_mov_b32_e32 v10, v12698; GFX7-NEXT: v_mov_b32_e32 v9, v02699; GFX7-NEXT: s_mov_b64 s[6:7], exec2700; GFX7-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=12701; GFX7-NEXT: v_readfirstlane_b32 s8, v92702; GFX7-NEXT: v_readfirstlane_b32 s9, v102703; GFX7-NEXT: v_readfirstlane_b32 s10, v72704; GFX7-NEXT: v_readfirstlane_b32 s11, v82705; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]2706; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]2707; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]2708; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]2709; GFX7-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:20482710; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]2711; GFX7-NEXT: s_cbranch_execnz .LBB10_12712; GFX7-NEXT: ; %bb.2:2713; GFX7-NEXT: s_mov_b64 exec, s[6:7]2714; GFX7-NEXT: s_mov_b64 s[6:7], 02715; GFX7-NEXT: .LBB10_3: ; %atomicrmw.start2716; GFX7-NEXT: ; =>This Loop Header: Depth=12717; GFX7-NEXT: ; Child Loop BB10_4 Depth 22718; GFX7-NEXT: s_waitcnt vmcnt(0)2719; GFX7-NEXT: v_add_f64 v[11:12], v[13:14], v[5:6]2720; GFX7-NEXT: s_mov_b64 s[12:13], exec2721; GFX7-NEXT: v_mov_b32_e32 v0, v112722; GFX7-NEXT: v_mov_b32_e32 v1, v122723; GFX7-NEXT: v_mov_b32_e32 v2, v132724; GFX7-NEXT: v_mov_b32_e32 v3, v142725; GFX7-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=12726; GFX7-NEXT: ; => This Inner Loop Header: Depth=22727; GFX7-NEXT: v_readfirstlane_b32 s8, v92728; GFX7-NEXT: v_readfirstlane_b32 s9, v102729; GFX7-NEXT: v_readfirstlane_b32 s10, v72730; GFX7-NEXT: v_readfirstlane_b32 s11, v82731; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]2732; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]2733; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]2734; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]2735; GFX7-NEXT: s_waitcnt vmcnt(0)2736; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v4, s[8:11], 0 offen offset:2048 glc2737; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]2738; GFX7-NEXT: s_cbranch_execnz .LBB10_42739; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=12740; GFX7-NEXT: s_mov_b64 exec, s[12:13]2741; GFX7-NEXT: s_waitcnt vmcnt(0)2742; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]2743; GFX7-NEXT: v_mov_b32_e32 v14, v12744; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]2745; GFX7-NEXT: v_mov_b32_e32 v13, v02746; GFX7-NEXT: buffer_wbinvl12747; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]2748; GFX7-NEXT: s_cbranch_execnz .LBB10_32749; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end2750; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]2751; GFX7-NEXT: s_setpc_b64 s[30:31]2752;2753; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__waterfall__amdgpu_no_fine_grained_memory:2754; GFX6: ; %bb.0:2755; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2756; GFX6-NEXT: v_mov_b32_e32 v8, v32757; GFX6-NEXT: v_mov_b32_e32 v7, v22758; GFX6-NEXT: v_mov_b32_e32 v10, v12759; GFX6-NEXT: v_mov_b32_e32 v9, v02760; GFX6-NEXT: v_add_i32_e32 v15, vcc, 0x800, v42761; GFX6-NEXT: s_mov_b64 s[6:7], exec2762; GFX6-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=12763; GFX6-NEXT: v_readfirstlane_b32 s8, v92764; GFX6-NEXT: v_readfirstlane_b32 s9, v102765; GFX6-NEXT: v_readfirstlane_b32 s10, v72766; GFX6-NEXT: v_readfirstlane_b32 s11, v82767; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]2768; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]2769; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]2770; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]2771; GFX6-NEXT: buffer_load_dwordx2 v[13:14], v4, s[8:11], 0 offen offset:20482772; GFX6-NEXT: ; implicit-def: $vgpr42773; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]2774; GFX6-NEXT: s_cbranch_execnz .LBB10_12775; GFX6-NEXT: ; %bb.2:2776; GFX6-NEXT: s_mov_b64 exec, s[6:7]2777; GFX6-NEXT: s_mov_b64 s[6:7], 02778; GFX6-NEXT: .LBB10_3: ; %atomicrmw.start2779; GFX6-NEXT: ; =>This Loop Header: Depth=12780; GFX6-NEXT: ; Child Loop BB10_4 Depth 22781; GFX6-NEXT: s_waitcnt vmcnt(0)2782; GFX6-NEXT: v_add_f64 v[11:12], v[13:14], v[5:6]2783; GFX6-NEXT: s_mov_b64 s[12:13], exec2784; GFX6-NEXT: s_waitcnt expcnt(0)2785; GFX6-NEXT: v_mov_b32_e32 v0, v112786; GFX6-NEXT: v_mov_b32_e32 v1, v122787; GFX6-NEXT: v_mov_b32_e32 v2, v132788; GFX6-NEXT: v_mov_b32_e32 v3, v142789; GFX6-NEXT: .LBB10_4: ; Parent Loop BB10_3 Depth=12790; GFX6-NEXT: ; => This Inner Loop Header: Depth=22791; GFX6-NEXT: v_readfirstlane_b32 s8, v92792; GFX6-NEXT: v_readfirstlane_b32 s9, v102793; GFX6-NEXT: v_readfirstlane_b32 s10, v72794; GFX6-NEXT: v_readfirstlane_b32 s11, v82795; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[9:10]2796; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[7:8]2797; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]2798; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]2799; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)2800; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v15, s[8:11], 0 offen glc2801; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]2802; GFX6-NEXT: s_cbranch_execnz .LBB10_42803; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB10_3 Depth=12804; GFX6-NEXT: s_mov_b64 exec, s[12:13]2805; GFX6-NEXT: s_waitcnt vmcnt(0)2806; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[13:14]2807; GFX6-NEXT: v_mov_b32_e32 v14, v12808; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]2809; GFX6-NEXT: v_mov_b32_e32 v13, v02810; GFX6-NEXT: buffer_wbinvl12811; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]2812; GFX6-NEXT: s_cbranch_execnz .LBB10_32813; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end2814; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]2815; GFX6-NEXT: s_waitcnt expcnt(0)2816; GFX6-NEXT: s_setpc_b64 s[30:31]2817 %gep = getelementptr double, ptr addrspace(7) %ptr, i32 2562818 %result = atomicrmw fadd ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !02819 ret double %result2820}2821 2822define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {2823; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:2824; GFX12: ; %bb.0:2825; GFX12-NEXT: s_wait_loadcnt_dscnt 0x02826; GFX12-NEXT: s_wait_expcnt 0x02827; GFX12-NEXT: s_wait_samplecnt 0x02828; GFX12-NEXT: s_wait_bvhcnt 0x02829; GFX12-NEXT: s_wait_kmcnt 0x02830; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v02831; GFX12-NEXT: v_mov_b32_e32 v0, s162832; GFX12-NEXT: v_mov_b32_e32 v10, s162833; GFX12-NEXT: s_mov_b32 s4, 02834; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:20482835; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start2836; GFX12-NEXT: ; =>This Inner Loop Header: Depth=12837; GFX12-NEXT: s_wait_loadcnt 0x02838; GFX12-NEXT: v_add_f64_e32 v[6:7], v[8:9], v[4:5]2839; GFX12-NEXT: s_wait_storecnt 0x02840; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)2841; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v72842; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v92843; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN2844; GFX12-NEXT: s_wait_loadcnt 0x02845; GFX12-NEXT: global_inv scope:SCOPE_DEV2846; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]2847; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v02848; GFX12-NEXT: s_or_b32 s4, vcc_lo, s42849; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)2850; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s42851; GFX12-NEXT: s_cbranch_execnz .LBB11_12852; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end2853; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s42854; GFX12-NEXT: s_wait_loadcnt 0x02855; GFX12-NEXT: s_setpc_b64 s[30:31]2856;2857; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:2858; GFX942: ; %bb.0:2859; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2860; GFX942-NEXT: v_mov_b32_e32 v2, s162861; GFX942-NEXT: buffer_wbl2 sc12862; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc02863; GFX942-NEXT: s_waitcnt vmcnt(0)2864; GFX942-NEXT: buffer_inv sc12865; GFX942-NEXT: s_setpc_b64 s[30:31]2866;2867; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:2868; GFX11: ; %bb.0:2869; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2870; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v02871; GFX11-NEXT: v_mov_b32_e32 v0, s162872; GFX11-NEXT: v_mov_b32_e32 v10, s162873; GFX11-NEXT: s_mov_b32 s4, 02874; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:20482875; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start2876; GFX11-NEXT: ; =>This Inner Loop Header: Depth=12877; GFX11-NEXT: s_waitcnt vmcnt(0)2878; GFX11-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]2879; GFX11-NEXT: s_waitcnt_vscnt null, 0x02880; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)2881; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v72882; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v92883; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc2884; GFX11-NEXT: s_waitcnt vmcnt(0)2885; GFX11-NEXT: buffer_gl1_inv2886; GFX11-NEXT: buffer_gl0_inv2887; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]2888; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v02889; GFX11-NEXT: s_or_b32 s4, vcc_lo, s42890; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2891; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s42892; GFX11-NEXT: s_cbranch_execnz .LBB11_12893; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end2894; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s42895; GFX11-NEXT: s_setpc_b64 s[30:31]2896;2897; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:2898; GFX10: ; %bb.0:2899; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2900; GFX10-NEXT: v_mov_b32_e32 v4, v02901; GFX10-NEXT: v_mov_b32_e32 v0, s202902; GFX10-NEXT: v_mov_b32_e32 v5, v12903; GFX10-NEXT: v_mov_b32_e32 v10, s202904; GFX10-NEXT: s_mov_b32 s4, 02905; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20482906; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start2907; GFX10-NEXT: ; =>This Inner Loop Header: Depth=12908; GFX10-NEXT: s_waitcnt vmcnt(0)2909; GFX10-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]2910; GFX10-NEXT: s_waitcnt_vscnt null, 0x02911; GFX10-NEXT: v_mov_b32_e32 v0, v62912; GFX10-NEXT: v_mov_b32_e32 v1, v72913; GFX10-NEXT: v_mov_b32_e32 v2, v82914; GFX10-NEXT: v_mov_b32_e32 v3, v92915; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc2916; GFX10-NEXT: s_waitcnt vmcnt(0)2917; GFX10-NEXT: buffer_gl1_inv2918; GFX10-NEXT: buffer_gl0_inv2919; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]2920; GFX10-NEXT: v_mov_b32_e32 v9, v12921; GFX10-NEXT: v_mov_b32_e32 v8, v02922; GFX10-NEXT: s_or_b32 s4, vcc_lo, s42923; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s42924; GFX10-NEXT: s_cbranch_execnz .LBB11_12925; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end2926; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s42927; GFX10-NEXT: s_setpc_b64 s[30:31]2928;2929; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:2930; GFX90A: ; %bb.0:2931; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2932; GFX90A-NEXT: v_mov_b32_e32 v4, v02933; GFX90A-NEXT: v_mov_b32_e32 v0, s202934; GFX90A-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20482935; GFX90A-NEXT: v_mov_b32_e32 v5, v12936; GFX90A-NEXT: s_mov_b64 s[4:5], 02937; GFX90A-NEXT: v_mov_b32_e32 v10, s202938; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start2939; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12940; GFX90A-NEXT: s_waitcnt vmcnt(0)2941; GFX90A-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]2942; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]2943; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[8:9], v[8:9] op_sel:[0,1]2944; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc2945; GFX90A-NEXT: s_waitcnt vmcnt(0)2946; GFX90A-NEXT: buffer_wbinvl12947; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2948; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2949; GFX90A-NEXT: v_pk_mov_b32 v[8:9], v[0:1], v[0:1] op_sel:[0,1]2950; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]2951; GFX90A-NEXT: s_cbranch_execnz .LBB11_12952; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end2953; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2954; GFX90A-NEXT: s_setpc_b64 s[30:31]2955;2956; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:2957; GFX908: ; %bb.0:2958; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2959; GFX908-NEXT: v_mov_b32_e32 v4, v02960; GFX908-NEXT: v_mov_b32_e32 v0, s202961; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20482962; GFX908-NEXT: v_mov_b32_e32 v5, v12963; GFX908-NEXT: s_mov_b64 s[4:5], 02964; GFX908-NEXT: v_mov_b32_e32 v10, s202965; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start2966; GFX908-NEXT: ; =>This Inner Loop Header: Depth=12967; GFX908-NEXT: s_waitcnt vmcnt(0)2968; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]2969; GFX908-NEXT: v_mov_b32_e32 v0, v62970; GFX908-NEXT: v_mov_b32_e32 v1, v72971; GFX908-NEXT: v_mov_b32_e32 v2, v82972; GFX908-NEXT: v_mov_b32_e32 v3, v92973; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc2974; GFX908-NEXT: s_waitcnt vmcnt(0)2975; GFX908-NEXT: buffer_wbinvl12976; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]2977; GFX908-NEXT: v_mov_b32_e32 v9, v12978; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2979; GFX908-NEXT: v_mov_b32_e32 v8, v02980; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]2981; GFX908-NEXT: s_cbranch_execnz .LBB11_12982; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end2983; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]2984; GFX908-NEXT: s_setpc_b64 s[30:31]2985;2986; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:2987; GFX8: ; %bb.0:2988; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2989; GFX8-NEXT: v_mov_b32_e32 v4, v02990; GFX8-NEXT: v_mov_b32_e32 v0, s202991; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20482992; GFX8-NEXT: v_mov_b32_e32 v5, v12993; GFX8-NEXT: s_mov_b64 s[4:5], 02994; GFX8-NEXT: v_mov_b32_e32 v10, s202995; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start2996; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12997; GFX8-NEXT: s_waitcnt vmcnt(0)2998; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]2999; GFX8-NEXT: v_mov_b32_e32 v0, v63000; GFX8-NEXT: v_mov_b32_e32 v1, v73001; GFX8-NEXT: v_mov_b32_e32 v2, v83002; GFX8-NEXT: v_mov_b32_e32 v3, v93003; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc3004; GFX8-NEXT: s_waitcnt vmcnt(0)3005; GFX8-NEXT: buffer_wbinvl13006; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3007; GFX8-NEXT: v_mov_b32_e32 v9, v13008; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3009; GFX8-NEXT: v_mov_b32_e32 v8, v03010; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3011; GFX8-NEXT: s_cbranch_execnz .LBB11_13012; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3013; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3014; GFX8-NEXT: s_setpc_b64 s[30:31]3015;3016; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:3017; GFX7: ; %bb.0:3018; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3019; GFX7-NEXT: v_mov_b32_e32 v4, v03020; GFX7-NEXT: v_mov_b32_e32 v0, s203021; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20483022; GFX7-NEXT: v_mov_b32_e32 v5, v13023; GFX7-NEXT: s_mov_b64 s[4:5], 03024; GFX7-NEXT: v_mov_b32_e32 v10, s203025; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start3026; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13027; GFX7-NEXT: s_waitcnt vmcnt(0)3028; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]3029; GFX7-NEXT: v_mov_b32_e32 v0, v63030; GFX7-NEXT: v_mov_b32_e32 v1, v73031; GFX7-NEXT: v_mov_b32_e32 v2, v83032; GFX7-NEXT: v_mov_b32_e32 v3, v93033; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc3034; GFX7-NEXT: s_waitcnt vmcnt(0)3035; GFX7-NEXT: buffer_wbinvl13036; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3037; GFX7-NEXT: v_mov_b32_e32 v9, v13038; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3039; GFX7-NEXT: v_mov_b32_e32 v8, v03040; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3041; GFX7-NEXT: s_cbranch_execnz .LBB11_13042; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3043; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3044; GFX7-NEXT: s_setpc_b64 s[30:31]3045;3046; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remote_memory:3047; GFX6: ; %bb.0:3048; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3049; GFX6-NEXT: v_mov_b32_e32 v4, v03050; GFX6-NEXT: v_mov_b32_e32 v0, s203051; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20483052; GFX6-NEXT: s_add_i32 s6, s20, 0x8003053; GFX6-NEXT: v_mov_b32_e32 v5, v13054; GFX6-NEXT: s_mov_b64 s[4:5], 03055; GFX6-NEXT: v_mov_b32_e32 v10, s63056; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start3057; GFX6-NEXT: ; =>This Inner Loop Header: Depth=13058; GFX6-NEXT: s_waitcnt vmcnt(0)3059; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]3060; GFX6-NEXT: s_waitcnt expcnt(0)3061; GFX6-NEXT: v_mov_b32_e32 v0, v63062; GFX6-NEXT: v_mov_b32_e32 v1, v73063; GFX6-NEXT: v_mov_b32_e32 v2, v83064; GFX6-NEXT: v_mov_b32_e32 v3, v93065; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc3066; GFX6-NEXT: s_waitcnt vmcnt(0)3067; GFX6-NEXT: buffer_wbinvl13068; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3069; GFX6-NEXT: v_mov_b32_e32 v9, v13070; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3071; GFX6-NEXT: v_mov_b32_e32 v8, v03072; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]3073; GFX6-NEXT: s_cbranch_execnz .LBB11_13074; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end3075; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]3076; GFX6-NEXT: s_waitcnt expcnt(0)3077; GFX6-NEXT: s_setpc_b64 s[30:31]3078 %gep = getelementptr double, ptr addrspace(7) %ptr, i32 2563079 %result = atomicrmw fadd ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !03080 ret double %result3081}3082 3083define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, double %val) #0 {3084; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:3085; GFX12: ; %bb.0:3086; GFX12-NEXT: s_wait_loadcnt_dscnt 0x03087; GFX12-NEXT: s_wait_expcnt 0x03088; GFX12-NEXT: s_wait_samplecnt 0x03089; GFX12-NEXT: s_wait_bvhcnt 0x03090; GFX12-NEXT: s_wait_kmcnt 0x03091; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v03092; GFX12-NEXT: v_mov_b32_e32 v0, s163093; GFX12-NEXT: v_mov_b32_e32 v10, s163094; GFX12-NEXT: s_mov_b32 s4, 03095; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:20483096; GFX12-NEXT: .LBB12_1: ; %atomicrmw.start3097; GFX12-NEXT: ; =>This Inner Loop Header: Depth=13098; GFX12-NEXT: s_wait_loadcnt 0x03099; GFX12-NEXT: v_add_f64_e32 v[6:7], v[8:9], v[4:5]3100; GFX12-NEXT: s_wait_storecnt 0x03101; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)3102; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v73103; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v93104; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN3105; GFX12-NEXT: s_wait_loadcnt 0x03106; GFX12-NEXT: global_inv scope:SCOPE_DEV3107; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]3108; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v03109; GFX12-NEXT: s_or_b32 s4, vcc_lo, s43110; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)3111; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s43112; GFX12-NEXT: s_cbranch_execnz .LBB12_13113; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end3114; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s43115; GFX12-NEXT: s_wait_loadcnt 0x03116; GFX12-NEXT: s_setpc_b64 s[30:31]3117;3118; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:3119; GFX942: ; %bb.0:3120; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3121; GFX942-NEXT: v_mov_b32_e32 v2, s163122; GFX942-NEXT: buffer_wbl2 sc13123; GFX942-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], 0 offen offset:2048 sc03124; GFX942-NEXT: s_waitcnt vmcnt(0)3125; GFX942-NEXT: buffer_inv sc13126; GFX942-NEXT: s_setpc_b64 s[30:31]3127;3128; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:3129; GFX11: ; %bb.0:3130; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3131; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v03132; GFX11-NEXT: v_mov_b32_e32 v0, s163133; GFX11-NEXT: v_mov_b32_e32 v10, s163134; GFX11-NEXT: s_mov_b32 s4, 03135; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:20483136; GFX11-NEXT: .LBB12_1: ; %atomicrmw.start3137; GFX11-NEXT: ; =>This Inner Loop Header: Depth=13138; GFX11-NEXT: s_waitcnt vmcnt(0)3139; GFX11-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]3140; GFX11-NEXT: s_waitcnt_vscnt null, 0x03141; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)3142; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v73143; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v93144; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc3145; GFX11-NEXT: s_waitcnt vmcnt(0)3146; GFX11-NEXT: buffer_gl1_inv3147; GFX11-NEXT: buffer_gl0_inv3148; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]3149; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v03150; GFX11-NEXT: s_or_b32 s4, vcc_lo, s43151; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3152; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s43153; GFX11-NEXT: s_cbranch_execnz .LBB12_13154; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end3155; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s43156; GFX11-NEXT: s_setpc_b64 s[30:31]3157;3158; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:3159; GFX10: ; %bb.0:3160; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3161; GFX10-NEXT: v_mov_b32_e32 v4, v03162; GFX10-NEXT: v_mov_b32_e32 v0, s203163; GFX10-NEXT: v_mov_b32_e32 v5, v13164; GFX10-NEXT: v_mov_b32_e32 v10, s203165; GFX10-NEXT: s_mov_b32 s4, 03166; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20483167; GFX10-NEXT: .LBB12_1: ; %atomicrmw.start3168; GFX10-NEXT: ; =>This Inner Loop Header: Depth=13169; GFX10-NEXT: s_waitcnt vmcnt(0)3170; GFX10-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]3171; GFX10-NEXT: s_waitcnt_vscnt null, 0x03172; GFX10-NEXT: v_mov_b32_e32 v0, v63173; GFX10-NEXT: v_mov_b32_e32 v1, v73174; GFX10-NEXT: v_mov_b32_e32 v2, v83175; GFX10-NEXT: v_mov_b32_e32 v3, v93176; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc3177; GFX10-NEXT: s_waitcnt vmcnt(0)3178; GFX10-NEXT: buffer_gl1_inv3179; GFX10-NEXT: buffer_gl0_inv3180; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]3181; GFX10-NEXT: v_mov_b32_e32 v9, v13182; GFX10-NEXT: v_mov_b32_e32 v8, v03183; GFX10-NEXT: s_or_b32 s4, vcc_lo, s43184; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s43185; GFX10-NEXT: s_cbranch_execnz .LBB12_13186; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end3187; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s43188; GFX10-NEXT: s_setpc_b64 s[30:31]3189;3190; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:3191; GFX90A: ; %bb.0:3192; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3193; GFX90A-NEXT: v_mov_b32_e32 v2, s203194; GFX90A-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[16:19], 0 offen offset:2048 glc3195; GFX90A-NEXT: s_waitcnt vmcnt(0)3196; GFX90A-NEXT: buffer_wbinvl13197; GFX90A-NEXT: s_setpc_b64 s[30:31]3198;3199; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:3200; GFX908: ; %bb.0:3201; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3202; GFX908-NEXT: v_mov_b32_e32 v4, v03203; GFX908-NEXT: v_mov_b32_e32 v0, s203204; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20483205; GFX908-NEXT: v_mov_b32_e32 v5, v13206; GFX908-NEXT: s_mov_b64 s[4:5], 03207; GFX908-NEXT: v_mov_b32_e32 v10, s203208; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start3209; GFX908-NEXT: ; =>This Inner Loop Header: Depth=13210; GFX908-NEXT: s_waitcnt vmcnt(0)3211; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]3212; GFX908-NEXT: v_mov_b32_e32 v0, v63213; GFX908-NEXT: v_mov_b32_e32 v1, v73214; GFX908-NEXT: v_mov_b32_e32 v2, v83215; GFX908-NEXT: v_mov_b32_e32 v3, v93216; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc3217; GFX908-NEXT: s_waitcnt vmcnt(0)3218; GFX908-NEXT: buffer_wbinvl13219; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3220; GFX908-NEXT: v_mov_b32_e32 v9, v13221; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3222; GFX908-NEXT: v_mov_b32_e32 v8, v03223; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]3224; GFX908-NEXT: s_cbranch_execnz .LBB12_13225; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end3226; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]3227; GFX908-NEXT: s_setpc_b64 s[30:31]3228;3229; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:3230; GFX8: ; %bb.0:3231; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3232; GFX8-NEXT: v_mov_b32_e32 v4, v03233; GFX8-NEXT: v_mov_b32_e32 v0, s203234; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20483235; GFX8-NEXT: v_mov_b32_e32 v5, v13236; GFX8-NEXT: s_mov_b64 s[4:5], 03237; GFX8-NEXT: v_mov_b32_e32 v10, s203238; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start3239; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13240; GFX8-NEXT: s_waitcnt vmcnt(0)3241; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]3242; GFX8-NEXT: v_mov_b32_e32 v0, v63243; GFX8-NEXT: v_mov_b32_e32 v1, v73244; GFX8-NEXT: v_mov_b32_e32 v2, v83245; GFX8-NEXT: v_mov_b32_e32 v3, v93246; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc3247; GFX8-NEXT: s_waitcnt vmcnt(0)3248; GFX8-NEXT: buffer_wbinvl13249; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3250; GFX8-NEXT: v_mov_b32_e32 v9, v13251; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3252; GFX8-NEXT: v_mov_b32_e32 v8, v03253; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3254; GFX8-NEXT: s_cbranch_execnz .LBB12_13255; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3256; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3257; GFX8-NEXT: s_setpc_b64 s[30:31]3258;3259; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:3260; GFX7: ; %bb.0:3261; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3262; GFX7-NEXT: v_mov_b32_e32 v4, v03263; GFX7-NEXT: v_mov_b32_e32 v0, s203264; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20483265; GFX7-NEXT: v_mov_b32_e32 v5, v13266; GFX7-NEXT: s_mov_b64 s[4:5], 03267; GFX7-NEXT: v_mov_b32_e32 v10, s203268; GFX7-NEXT: .LBB12_1: ; %atomicrmw.start3269; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13270; GFX7-NEXT: s_waitcnt vmcnt(0)3271; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]3272; GFX7-NEXT: v_mov_b32_e32 v0, v63273; GFX7-NEXT: v_mov_b32_e32 v1, v73274; GFX7-NEXT: v_mov_b32_e32 v2, v83275; GFX7-NEXT: v_mov_b32_e32 v3, v93276; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc3277; GFX7-NEXT: s_waitcnt vmcnt(0)3278; GFX7-NEXT: buffer_wbinvl13279; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3280; GFX7-NEXT: v_mov_b32_e32 v9, v13281; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3282; GFX7-NEXT: v_mov_b32_e32 v8, v03283; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3284; GFX7-NEXT: s_cbranch_execnz .LBB12_13285; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3286; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3287; GFX7-NEXT: s_setpc_b64 s[30:31]3288;3289; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:3290; GFX6: ; %bb.0:3291; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3292; GFX6-NEXT: v_mov_b32_e32 v4, v03293; GFX6-NEXT: v_mov_b32_e32 v0, s203294; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:20483295; GFX6-NEXT: s_add_i32 s6, s20, 0x8003296; GFX6-NEXT: v_mov_b32_e32 v5, v13297; GFX6-NEXT: s_mov_b64 s[4:5], 03298; GFX6-NEXT: v_mov_b32_e32 v10, s63299; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start3300; GFX6-NEXT: ; =>This Inner Loop Header: Depth=13301; GFX6-NEXT: s_waitcnt vmcnt(0)3302; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]3303; GFX6-NEXT: s_waitcnt expcnt(0)3304; GFX6-NEXT: v_mov_b32_e32 v0, v63305; GFX6-NEXT: v_mov_b32_e32 v1, v73306; GFX6-NEXT: v_mov_b32_e32 v2, v83307; GFX6-NEXT: v_mov_b32_e32 v3, v93308; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc3309; GFX6-NEXT: s_waitcnt vmcnt(0)3310; GFX6-NEXT: buffer_wbinvl13311; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]3312; GFX6-NEXT: v_mov_b32_e32 v9, v13313; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3314; GFX6-NEXT: v_mov_b32_e32 v8, v03315; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]3316; GFX6-NEXT: s_cbranch_execnz .LBB12_13317; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end3318; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]3319; GFX6-NEXT: s_waitcnt expcnt(0)3320; GFX6-NEXT: s_setpc_b64 s[30:31]3321 %gep = getelementptr double, ptr addrspace(7) %ptr, i32 2563322 %result = atomicrmw fadd ptr addrspace(7) %gep, double %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !03323 ret double %result3324}3325 3326; --------------------------------------------------------------------3327; half3328; --------------------------------------------------------------------3329 3330define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, half %val) #0 {3331; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3332; GFX12-TRUE16: ; %bb.0:3333; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x03334; GFX12-TRUE16-NEXT: s_wait_expcnt 0x03335; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x03336; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x03337; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x03338; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x2003339; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3340; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -43341; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3342; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s43343; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 33344; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3345; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 33346; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3347; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s43348; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen3349; GFX12-TRUE16-NEXT: s_not_b32 s6, s53350; GFX12-TRUE16-NEXT: s_mov_b32 s5, 03351; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start3352; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=13353; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x03354; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v23355; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 03356; GFX12-TRUE16-NEXT: s_wait_storecnt 0x03357; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3358; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, v1.l, v0.l3359; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v13360; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3361; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3362; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v13363; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v13364; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN3365; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x03366; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV3367; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v23368; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v33369; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s53370; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3371; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s53372; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_13373; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end3374; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s53375; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v33376; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x03377; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]3378;3379; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3380; GFX12-FAKE16: ; %bb.0:3381; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x03382; GFX12-FAKE16-NEXT: s_wait_expcnt 0x03383; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x03384; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x03385; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x03386; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x2003387; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3388; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -43389; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3390; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, s43391; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 33392; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3393; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 33394; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3395; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s43396; GFX12-FAKE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen3397; GFX12-FAKE16-NEXT: s_not_b32 s6, s53398; GFX12-FAKE16-NEXT: s_mov_b32 s5, 03399; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start3400; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=13401; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x03402; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s4, v23403; GFX12-FAKE16-NEXT: s_wait_storecnt 0x03404; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3405; GFX12-FAKE16-NEXT: v_add_f16_e32 v1, v1, v03406; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v13407; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)3408; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s4, v13409; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3410; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s6, v13411; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3412; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v13413; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN3414; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x03415; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV3416; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v23417; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v33418; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s53419; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3420; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s53421; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_13422; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end3423; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s53424; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v33425; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x03426; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]3427;3428; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3429; GFX942: ; %bb.0:3430; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3431; GFX942-NEXT: s_addk_i32 s16, 0x2003432; GFX942-NEXT: s_and_b32 s4, s16, -43433; GFX942-NEXT: v_mov_b32_e32 v1, s43434; GFX942-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen3435; GFX942-NEXT: s_and_b32 s4, s16, 33436; GFX942-NEXT: s_lshl_b32 s6, s4, 33437; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s63438; GFX942-NEXT: s_not_b32 s7, s43439; GFX942-NEXT: s_mov_b64 s[4:5], 03440; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start3441; GFX942-NEXT: ; =>This Inner Loop Header: Depth=13442; GFX942-NEXT: s_waitcnt vmcnt(0)3443; GFX942-NEXT: v_lshrrev_b32_e32 v2, s6, v33444; GFX942-NEXT: v_add_f16_e32 v2, v2, v03445; GFX942-NEXT: v_lshlrev_b32_e32 v2, s6, v23446; GFX942-NEXT: v_and_or_b32 v2, v3, s7, v23447; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]3448; GFX942-NEXT: buffer_wbl2 sc13449; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[0:3], 0 offen sc03450; GFX942-NEXT: s_waitcnt vmcnt(0)3451; GFX942-NEXT: buffer_inv sc13452; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v33453; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3454; GFX942-NEXT: v_mov_b32_e32 v3, v43455; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]3456; GFX942-NEXT: s_cbranch_execnz .LBB13_13457; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end3458; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]3459; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v43460; GFX942-NEXT: s_setpc_b64 s[30:31]3461;3462; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3463; GFX11-TRUE16: ; %bb.0:3464; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3465; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x2003466; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3467; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -43468; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s43469; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 33470; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3471; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 33472; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s43473; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen3474; GFX11-TRUE16-NEXT: s_not_b32 s6, s53475; GFX11-TRUE16-NEXT: s_mov_b32 s5, 03476; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start3477; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=13478; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)3479; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v23480; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 03481; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x03482; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3483; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, v1.l, v0.l3484; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v13485; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3486; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v13487; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v13488; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc3489; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)3490; GFX11-TRUE16-NEXT: buffer_gl1_inv3491; GFX11-TRUE16-NEXT: buffer_gl0_inv3492; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v23493; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v33494; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s53495; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3496; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s53497; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_13498; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end3499; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s53500; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v33501; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]3502;3503; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3504; GFX11-FAKE16: ; %bb.0:3505; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3506; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x2003507; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3508; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -43509; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s43510; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 33511; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3512; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 33513; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s43514; GFX11-FAKE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen3515; GFX11-FAKE16-NEXT: s_not_b32 s6, s53516; GFX11-FAKE16-NEXT: s_mov_b32 s5, 03517; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start3518; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=13519; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)3520; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s4, v23521; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x03522; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3523; GFX11-FAKE16-NEXT: v_add_f16_e32 v1, v1, v03524; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v13525; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3526; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s4, v13527; GFX11-FAKE16-NEXT: v_and_or_b32 v1, v2, s6, v13528; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3529; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v13530; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc3531; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)3532; GFX11-FAKE16-NEXT: buffer_gl1_inv3533; GFX11-FAKE16-NEXT: buffer_gl0_inv3534; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v23535; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v33536; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s53537; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3538; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s53539; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_13540; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end3541; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s53542; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v33543; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]3544;3545; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3546; GFX10: ; %bb.0:3547; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3548; GFX10-NEXT: s_addk_i32 s20, 0x2003549; GFX10-NEXT: s_and_b32 s4, s20, -43550; GFX10-NEXT: v_mov_b32_e32 v5, s43551; GFX10-NEXT: s_and_b32 s4, s20, 33552; GFX10-NEXT: s_lshl_b32 s4, s4, 33553; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s43554; GFX10-NEXT: buffer_load_dword v2, v5, s[16:19], 0 offen3555; GFX10-NEXT: s_not_b32 s6, s53556; GFX10-NEXT: s_mov_b32 s5, 03557; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start3558; GFX10-NEXT: ; =>This Inner Loop Header: Depth=13559; GFX10-NEXT: s_waitcnt vmcnt(0)3560; GFX10-NEXT: v_lshrrev_b32_e32 v1, s4, v23561; GFX10-NEXT: s_waitcnt_vscnt null, 0x03562; GFX10-NEXT: v_add_f16_e32 v1, v1, v03563; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_03564; GFX10-NEXT: v_and_or_b32 v1, v2, s6, v13565; GFX10-NEXT: v_mov_b32_e32 v4, v23566; GFX10-NEXT: v_mov_b32_e32 v3, v13567; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc3568; GFX10-NEXT: s_waitcnt vmcnt(0)3569; GFX10-NEXT: buffer_gl1_inv3570; GFX10-NEXT: buffer_gl0_inv3571; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v23572; GFX10-NEXT: v_mov_b32_e32 v2, v33573; GFX10-NEXT: s_or_b32 s5, vcc_lo, s53574; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s53575; GFX10-NEXT: s_cbranch_execnz .LBB13_13576; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end3577; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s53578; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v33579; GFX10-NEXT: s_setpc_b64 s[30:31]3580;3581; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3582; GFX90A: ; %bb.0:3583; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3584; GFX90A-NEXT: s_addk_i32 s20, 0x2003585; GFX90A-NEXT: s_and_b32 s4, s20, -43586; GFX90A-NEXT: v_mov_b32_e32 v1, s43587; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen3588; GFX90A-NEXT: s_and_b32 s4, s20, 33589; GFX90A-NEXT: s_lshl_b32 s6, s4, 33590; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s63591; GFX90A-NEXT: s_not_b32 s7, s43592; GFX90A-NEXT: s_mov_b64 s[4:5], 03593; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start3594; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=13595; GFX90A-NEXT: s_waitcnt vmcnt(0)3596; GFX90A-NEXT: v_lshrrev_b32_e32 v2, s6, v33597; GFX90A-NEXT: v_add_f16_e32 v2, v2, v03598; GFX90A-NEXT: v_lshlrev_b32_e32 v2, s6, v23599; GFX90A-NEXT: v_and_or_b32 v2, v3, s7, v23600; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]3601; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen glc3602; GFX90A-NEXT: s_waitcnt vmcnt(0)3603; GFX90A-NEXT: buffer_wbinvl13604; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v33605; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3606; GFX90A-NEXT: v_mov_b32_e32 v3, v43607; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]3608; GFX90A-NEXT: s_cbranch_execnz .LBB13_13609; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end3610; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3611; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v43612; GFX90A-NEXT: s_setpc_b64 s[30:31]3613;3614; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3615; GFX908: ; %bb.0:3616; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3617; GFX908-NEXT: s_addk_i32 s20, 0x2003618; GFX908-NEXT: s_and_b32 s4, s20, -43619; GFX908-NEXT: v_mov_b32_e32 v5, s43620; GFX908-NEXT: buffer_load_dword v2, v5, s[16:19], 0 offen3621; GFX908-NEXT: s_and_b32 s4, s20, 33622; GFX908-NEXT: s_lshl_b32 s6, s4, 33623; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s63624; GFX908-NEXT: s_not_b32 s7, s43625; GFX908-NEXT: s_mov_b64 s[4:5], 03626; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start3627; GFX908-NEXT: ; =>This Inner Loop Header: Depth=13628; GFX908-NEXT: s_waitcnt vmcnt(0)3629; GFX908-NEXT: v_lshrrev_b32_e32 v1, s6, v23630; GFX908-NEXT: v_add_f16_e32 v1, v1, v03631; GFX908-NEXT: v_lshlrev_b32_e32 v1, s6, v13632; GFX908-NEXT: v_and_or_b32 v1, v2, s7, v13633; GFX908-NEXT: v_mov_b32_e32 v4, v23634; GFX908-NEXT: v_mov_b32_e32 v3, v13635; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc3636; GFX908-NEXT: s_waitcnt vmcnt(0)3637; GFX908-NEXT: buffer_wbinvl13638; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v23639; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3640; GFX908-NEXT: v_mov_b32_e32 v2, v33641; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]3642; GFX908-NEXT: s_cbranch_execnz .LBB13_13643; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end3644; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]3645; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v33646; GFX908-NEXT: s_setpc_b64 s[30:31]3647;3648; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3649; GFX8: ; %bb.0:3650; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3651; GFX8-NEXT: s_addk_i32 s20, 0x2003652; GFX8-NEXT: s_and_b32 s4, s20, -43653; GFX8-NEXT: v_mov_b32_e32 v5, s43654; GFX8-NEXT: buffer_load_dword v2, v5, s[16:19], 0 offen3655; GFX8-NEXT: s_and_b32 s4, s20, 33656; GFX8-NEXT: s_lshl_b32 s6, s4, 33657; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s63658; GFX8-NEXT: s_not_b32 s7, s43659; GFX8-NEXT: s_mov_b64 s[4:5], 03660; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start3661; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13662; GFX8-NEXT: s_waitcnt vmcnt(0)3663; GFX8-NEXT: v_lshrrev_b32_e32 v1, s6, v23664; GFX8-NEXT: v_add_f16_e32 v1, v1, v03665; GFX8-NEXT: v_and_b32_e32 v3, s7, v23666; GFX8-NEXT: v_lshlrev_b32_e32 v1, s6, v13667; GFX8-NEXT: v_or_b32_e32 v1, v3, v13668; GFX8-NEXT: v_mov_b32_e32 v4, v23669; GFX8-NEXT: v_mov_b32_e32 v3, v13670; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc3671; GFX8-NEXT: s_waitcnt vmcnt(0)3672; GFX8-NEXT: buffer_wbinvl13673; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v23674; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3675; GFX8-NEXT: v_mov_b32_e32 v2, v33676; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3677; GFX8-NEXT: s_cbranch_execnz .LBB13_13678; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3679; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3680; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v33681; GFX8-NEXT: s_setpc_b64 s[30:31]3682;3683; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3684; GFX7: ; %bb.0:3685; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3686; GFX7-NEXT: s_addk_i32 s20, 0x2003687; GFX7-NEXT: s_and_b32 s4, s20, -43688; GFX7-NEXT: v_mov_b32_e32 v4, s43689; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen3690; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v03691; GFX7-NEXT: s_and_b32 s4, s20, 33692; GFX7-NEXT: s_lshl_b32 s6, s4, 33693; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s63694; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v03695; GFX7-NEXT: s_not_b32 s7, s43696; GFX7-NEXT: s_mov_b64 s[4:5], 03697; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start3698; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13699; GFX7-NEXT: s_waitcnt vmcnt(0)3700; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v13701; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v03702; GFX7-NEXT: v_and_b32_e32 v2, s7, v13703; GFX7-NEXT: v_add_f32_e32 v0, v0, v53704; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v03705; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v03706; GFX7-NEXT: v_or_b32_e32 v0, v2, v03707; GFX7-NEXT: v_mov_b32_e32 v3, v13708; GFX7-NEXT: v_mov_b32_e32 v2, v03709; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc3710; GFX7-NEXT: s_waitcnt vmcnt(0)3711; GFX7-NEXT: buffer_wbinvl13712; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v13713; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3714; GFX7-NEXT: v_mov_b32_e32 v1, v23715; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3716; GFX7-NEXT: s_cbranch_execnz .LBB13_13717; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3718; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3719; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v23720; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v03721; GFX7-NEXT: s_setpc_b64 s[30:31]3722;3723; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:3724; GFX6: ; %bb.0:3725; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3726; GFX6-NEXT: s_addk_i32 s20, 0x2003727; GFX6-NEXT: s_and_b32 s4, s20, -43728; GFX6-NEXT: v_mov_b32_e32 v4, s43729; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen3730; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v03731; GFX6-NEXT: s_and_b32 s4, s20, 33732; GFX6-NEXT: s_lshl_b32 s6, s4, 33733; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s63734; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v03735; GFX6-NEXT: s_not_b32 s7, s43736; GFX6-NEXT: s_mov_b64 s[4:5], 03737; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start3738; GFX6-NEXT: ; =>This Inner Loop Header: Depth=13739; GFX6-NEXT: s_waitcnt vmcnt(0)3740; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v13741; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v03742; GFX6-NEXT: s_waitcnt expcnt(0)3743; GFX6-NEXT: v_and_b32_e32 v2, s7, v13744; GFX6-NEXT: v_add_f32_e32 v0, v0, v53745; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v03746; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v03747; GFX6-NEXT: v_or_b32_e32 v0, v2, v03748; GFX6-NEXT: v_mov_b32_e32 v3, v13749; GFX6-NEXT: v_mov_b32_e32 v2, v03750; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc3751; GFX6-NEXT: s_waitcnt vmcnt(0)3752; GFX6-NEXT: buffer_wbinvl13753; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v13754; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3755; GFX6-NEXT: v_mov_b32_e32 v1, v23756; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]3757; GFX6-NEXT: s_cbranch_execnz .LBB13_13758; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end3759; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]3760; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v23761; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v03762; GFX6-NEXT: s_waitcnt expcnt(0)3763; GFX6-NEXT: s_setpc_b64 s[30:31]3764 %gep = getelementptr half, ptr addrspace(7) %ptr, i32 2563765 %result = atomicrmw fadd ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !03766 ret half %result3767}3768 3769define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, half %val) #0 {3770; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:3771; GFX12-TRUE16: ; %bb.0:3772; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x03773; GFX12-TRUE16-NEXT: s_wait_expcnt 0x03774; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x03775; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x03776; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x03777; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x2003778; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3779; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -43780; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3781; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s43782; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 33783; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3784; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 33785; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3786; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s43787; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen3788; GFX12-TRUE16-NEXT: s_not_b32 s6, s53789; GFX12-TRUE16-NEXT: s_mov_b32 s5, 03790; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start3791; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=13792; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x03793; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v23794; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 03795; GFX12-TRUE16-NEXT: s_wait_storecnt 0x03796; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3797; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, v1.l, v0.l3798; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v13799; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3800; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3801; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v13802; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v13803; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN3804; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x03805; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV3806; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v23807; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v43808; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s53809; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3810; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s53811; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_13812; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end3813; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s53814; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x03815; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]3816;3817; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:3818; GFX12-FAKE16: ; %bb.0:3819; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x03820; GFX12-FAKE16-NEXT: s_wait_expcnt 0x03821; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x03822; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x03823; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x03824; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x2003825; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3826; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -43827; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3828; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, s43829; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 33830; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3831; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 33832; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3833; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s43834; GFX12-FAKE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen3835; GFX12-FAKE16-NEXT: s_not_b32 s6, s53836; GFX12-FAKE16-NEXT: s_mov_b32 s5, 03837; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start3838; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=13839; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x03840; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s4, v23841; GFX12-FAKE16-NEXT: s_wait_storecnt 0x03842; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3843; GFX12-FAKE16-NEXT: v_add_f16_e32 v1, v1, v03844; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v13845; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)3846; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s4, v13847; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3848; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s6, v13849; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3850; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v13851; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN3852; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x03853; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV3854; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v23855; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v43856; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s53857; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3858; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s53859; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_13860; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end3861; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s53862; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x03863; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]3864;3865; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:3866; GFX942: ; %bb.0:3867; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3868; GFX942-NEXT: s_addk_i32 s16, 0x2003869; GFX942-NEXT: s_and_b32 s4, s16, -43870; GFX942-NEXT: v_mov_b32_e32 v1, s43871; GFX942-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen3872; GFX942-NEXT: s_and_b32 s4, s16, 33873; GFX942-NEXT: s_lshl_b32 s6, s4, 33874; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s63875; GFX942-NEXT: s_not_b32 s7, s43876; GFX942-NEXT: s_mov_b64 s[4:5], 03877; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start3878; GFX942-NEXT: ; =>This Inner Loop Header: Depth=13879; GFX942-NEXT: s_waitcnt vmcnt(0)3880; GFX942-NEXT: v_lshrrev_b32_e32 v2, s6, v33881; GFX942-NEXT: v_add_f16_e32 v2, v2, v03882; GFX942-NEXT: v_lshlrev_b32_e32 v2, s6, v23883; GFX942-NEXT: v_and_or_b32 v2, v3, s7, v23884; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]3885; GFX942-NEXT: buffer_wbl2 sc13886; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[0:3], 0 offen sc03887; GFX942-NEXT: s_waitcnt vmcnt(0)3888; GFX942-NEXT: buffer_inv sc13889; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v33890; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3891; GFX942-NEXT: v_mov_b32_e32 v3, v43892; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]3893; GFX942-NEXT: s_cbranch_execnz .LBB14_13894; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end3895; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]3896; GFX942-NEXT: s_setpc_b64 s[30:31]3897;3898; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:3899; GFX11-TRUE16: ; %bb.0:3900; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3901; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x2003902; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3903; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -43904; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s43905; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 33906; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3907; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 33908; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s43909; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen3910; GFX11-TRUE16-NEXT: s_not_b32 s6, s53911; GFX11-TRUE16-NEXT: s_mov_b32 s5, 03912; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start3913; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=13914; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)3915; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v23916; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 03917; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x03918; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3919; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, v1.l, v0.l3920; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v13921; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3922; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v13923; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v13924; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc3925; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)3926; GFX11-TRUE16-NEXT: buffer_gl1_inv3927; GFX11-TRUE16-NEXT: buffer_gl0_inv3928; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v23929; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v43930; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s53931; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3932; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s53933; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_13934; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end3935; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s53936; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]3937;3938; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:3939; GFX11-FAKE16: ; %bb.0:3940; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3941; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x2003942; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3943; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -43944; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s43945; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 33946; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)3947; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 33948; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s43949; GFX11-FAKE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen3950; GFX11-FAKE16-NEXT: s_not_b32 s6, s53951; GFX11-FAKE16-NEXT: s_mov_b32 s5, 03952; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start3953; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=13954; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)3955; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s4, v23956; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x03957; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3958; GFX11-FAKE16-NEXT: v_add_f16_e32 v1, v1, v03959; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v13960; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3961; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s4, v13962; GFX11-FAKE16-NEXT: v_and_or_b32 v1, v2, s6, v13963; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3964; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v13965; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc3966; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)3967; GFX11-FAKE16-NEXT: buffer_gl1_inv3968; GFX11-FAKE16-NEXT: buffer_gl0_inv3969; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v23970; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v43971; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s53972; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3973; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s53974; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_13975; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end3976; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s53977; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]3978;3979; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:3980; GFX10: ; %bb.0:3981; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3982; GFX10-NEXT: s_addk_i32 s20, 0x2003983; GFX10-NEXT: s_and_b32 s4, s20, -43984; GFX10-NEXT: v_mov_b32_e32 v3, s43985; GFX10-NEXT: s_and_b32 s4, s20, 33986; GFX10-NEXT: s_lshl_b32 s4, s4, 33987; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s43988; GFX10-NEXT: buffer_load_dword v2, v3, s[16:19], 0 offen3989; GFX10-NEXT: s_not_b32 s6, s53990; GFX10-NEXT: s_mov_b32 s5, 03991; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start3992; GFX10-NEXT: ; =>This Inner Loop Header: Depth=13993; GFX10-NEXT: s_waitcnt vmcnt(0)3994; GFX10-NEXT: v_lshrrev_b32_e32 v1, s4, v23995; GFX10-NEXT: s_waitcnt_vscnt null, 0x03996; GFX10-NEXT: v_add_f16_e32 v1, v1, v03997; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_03998; GFX10-NEXT: v_and_or_b32 v1, v2, s6, v13999; GFX10-NEXT: v_mov_b32_e32 v5, v24000; GFX10-NEXT: v_mov_b32_e32 v4, v14001; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc4002; GFX10-NEXT: s_waitcnt vmcnt(0)4003; GFX10-NEXT: buffer_gl1_inv4004; GFX10-NEXT: buffer_gl0_inv4005; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v24006; GFX10-NEXT: v_mov_b32_e32 v2, v44007; GFX10-NEXT: s_or_b32 s5, vcc_lo, s54008; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s54009; GFX10-NEXT: s_cbranch_execnz .LBB14_14010; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end4011; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s54012; GFX10-NEXT: s_setpc_b64 s[30:31]4013;4014; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:4015; GFX90A: ; %bb.0:4016; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4017; GFX90A-NEXT: s_addk_i32 s20, 0x2004018; GFX90A-NEXT: s_and_b32 s4, s20, -44019; GFX90A-NEXT: v_mov_b32_e32 v1, s44020; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen4021; GFX90A-NEXT: s_and_b32 s4, s20, 34022; GFX90A-NEXT: s_lshl_b32 s6, s4, 34023; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s64024; GFX90A-NEXT: s_not_b32 s7, s44025; GFX90A-NEXT: s_mov_b64 s[4:5], 04026; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start4027; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=14028; GFX90A-NEXT: s_waitcnt vmcnt(0)4029; GFX90A-NEXT: v_lshrrev_b32_e32 v2, s6, v34030; GFX90A-NEXT: v_add_f16_e32 v2, v2, v04031; GFX90A-NEXT: v_lshlrev_b32_e32 v2, s6, v24032; GFX90A-NEXT: v_and_or_b32 v2, v3, s7, v24033; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]4034; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen glc4035; GFX90A-NEXT: s_waitcnt vmcnt(0)4036; GFX90A-NEXT: buffer_wbinvl14037; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v34038; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4039; GFX90A-NEXT: v_mov_b32_e32 v3, v44040; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]4041; GFX90A-NEXT: s_cbranch_execnz .LBB14_14042; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end4043; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4044; GFX90A-NEXT: s_setpc_b64 s[30:31]4045;4046; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:4047; GFX908: ; %bb.0:4048; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4049; GFX908-NEXT: s_addk_i32 s20, 0x2004050; GFX908-NEXT: s_and_b32 s4, s20, -44051; GFX908-NEXT: v_mov_b32_e32 v3, s44052; GFX908-NEXT: buffer_load_dword v2, v3, s[16:19], 0 offen4053; GFX908-NEXT: s_and_b32 s4, s20, 34054; GFX908-NEXT: s_lshl_b32 s6, s4, 34055; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s64056; GFX908-NEXT: s_not_b32 s7, s44057; GFX908-NEXT: s_mov_b64 s[4:5], 04058; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start4059; GFX908-NEXT: ; =>This Inner Loop Header: Depth=14060; GFX908-NEXT: s_waitcnt vmcnt(0)4061; GFX908-NEXT: v_lshrrev_b32_e32 v1, s6, v24062; GFX908-NEXT: v_add_f16_e32 v1, v1, v04063; GFX908-NEXT: v_lshlrev_b32_e32 v1, s6, v14064; GFX908-NEXT: v_and_or_b32 v1, v2, s7, v14065; GFX908-NEXT: v_mov_b32_e32 v5, v24066; GFX908-NEXT: v_mov_b32_e32 v4, v14067; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc4068; GFX908-NEXT: s_waitcnt vmcnt(0)4069; GFX908-NEXT: buffer_wbinvl14070; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v24071; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4072; GFX908-NEXT: v_mov_b32_e32 v2, v44073; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]4074; GFX908-NEXT: s_cbranch_execnz .LBB14_14075; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end4076; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]4077; GFX908-NEXT: s_setpc_b64 s[30:31]4078;4079; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:4080; GFX8: ; %bb.0:4081; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4082; GFX8-NEXT: s_addk_i32 s20, 0x2004083; GFX8-NEXT: s_and_b32 s4, s20, -44084; GFX8-NEXT: v_mov_b32_e32 v3, s44085; GFX8-NEXT: buffer_load_dword v2, v3, s[16:19], 0 offen4086; GFX8-NEXT: s_and_b32 s4, s20, 34087; GFX8-NEXT: s_lshl_b32 s6, s4, 34088; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s64089; GFX8-NEXT: s_not_b32 s7, s44090; GFX8-NEXT: s_mov_b64 s[4:5], 04091; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start4092; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14093; GFX8-NEXT: s_waitcnt vmcnt(0)4094; GFX8-NEXT: v_lshrrev_b32_e32 v1, s6, v24095; GFX8-NEXT: v_add_f16_e32 v1, v1, v04096; GFX8-NEXT: v_and_b32_e32 v4, s7, v24097; GFX8-NEXT: v_lshlrev_b32_e32 v1, s6, v14098; GFX8-NEXT: v_or_b32_e32 v1, v4, v14099; GFX8-NEXT: v_mov_b32_e32 v5, v24100; GFX8-NEXT: v_mov_b32_e32 v4, v14101; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc4102; GFX8-NEXT: s_waitcnt vmcnt(0)4103; GFX8-NEXT: buffer_wbinvl14104; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v24105; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4106; GFX8-NEXT: v_mov_b32_e32 v2, v44107; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]4108; GFX8-NEXT: s_cbranch_execnz .LBB14_14109; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4110; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]4111; GFX8-NEXT: s_setpc_b64 s[30:31]4112;4113; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:4114; GFX7: ; %bb.0:4115; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4116; GFX7-NEXT: s_addk_i32 s20, 0x2004117; GFX7-NEXT: s_and_b32 s4, s20, -44118; GFX7-NEXT: v_mov_b32_e32 v2, s44119; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen4120; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v04121; GFX7-NEXT: s_and_b32 s4, s20, 34122; GFX7-NEXT: s_lshl_b32 s6, s4, 34123; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s64124; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v04125; GFX7-NEXT: s_not_b32 s7, s44126; GFX7-NEXT: s_mov_b64 s[4:5], 04127; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start4128; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14129; GFX7-NEXT: s_waitcnt vmcnt(0)4130; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v14131; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v04132; GFX7-NEXT: v_and_b32_e32 v4, s7, v14133; GFX7-NEXT: v_add_f32_e32 v0, v0, v34134; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v04135; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v04136; GFX7-NEXT: v_or_b32_e32 v0, v4, v04137; GFX7-NEXT: v_mov_b32_e32 v5, v14138; GFX7-NEXT: v_mov_b32_e32 v4, v04139; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc4140; GFX7-NEXT: s_waitcnt vmcnt(0)4141; GFX7-NEXT: buffer_wbinvl14142; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v14143; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4144; GFX7-NEXT: v_mov_b32_e32 v1, v44145; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]4146; GFX7-NEXT: s_cbranch_execnz .LBB14_14147; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4148; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]4149; GFX7-NEXT: s_setpc_b64 s[30:31]4150;4151; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:4152; GFX6: ; %bb.0:4153; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4154; GFX6-NEXT: s_addk_i32 s20, 0x2004155; GFX6-NEXT: s_and_b32 s4, s20, -44156; GFX6-NEXT: v_mov_b32_e32 v2, s44157; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen4158; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v04159; GFX6-NEXT: s_and_b32 s4, s20, 34160; GFX6-NEXT: s_lshl_b32 s6, s4, 34161; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s64162; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v04163; GFX6-NEXT: s_not_b32 s7, s44164; GFX6-NEXT: s_mov_b64 s[4:5], 04165; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start4166; GFX6-NEXT: ; =>This Inner Loop Header: Depth=14167; GFX6-NEXT: s_waitcnt vmcnt(0)4168; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v14169; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v04170; GFX6-NEXT: s_waitcnt expcnt(0)4171; GFX6-NEXT: v_and_b32_e32 v4, s7, v14172; GFX6-NEXT: v_add_f32_e32 v0, v0, v34173; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v04174; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v04175; GFX6-NEXT: v_or_b32_e32 v0, v4, v04176; GFX6-NEXT: v_mov_b32_e32 v5, v14177; GFX6-NEXT: v_mov_b32_e32 v4, v04178; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc4179; GFX6-NEXT: s_waitcnt vmcnt(0)4180; GFX6-NEXT: buffer_wbinvl14181; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v14182; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4183; GFX6-NEXT: v_mov_b32_e32 v1, v44184; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]4185; GFX6-NEXT: s_cbranch_execnz .LBB14_14186; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end4187; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]4188; GFX6-NEXT: s_waitcnt expcnt(0)4189; GFX6-NEXT: s_setpc_b64 s[30:31]4190 %gep = getelementptr half, ptr addrspace(7) %ptr, i32 2564191 %unused = atomicrmw fadd ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04192 ret void4193}4194 4195define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, half %val) #0 {4196; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4197; GFX12-TRUE16: ; %bb.0:4198; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x04199; GFX12-TRUE16-NEXT: s_wait_expcnt 0x04200; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x04201; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x04202; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x04203; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v44204; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo4205; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)4206; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v64207; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v64208; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v44209; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4210; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff4211; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v74212; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14213; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v04214; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v14215; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v24216; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v34217; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)4218; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)4219; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]4220; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]4221; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s04222; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)4223; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s04224; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x04225; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen4226; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s04227; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_14228; GFX12-TRUE16-NEXT: ; %bb.2:4229; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s14230; GFX12-TRUE16-NEXT: s_mov_b32 s1, 04231; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start4232; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=14233; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 24234; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x04235; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v74236; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, 04237; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo4238; GFX12-TRUE16-NEXT: s_wait_storecnt 0x04239; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)4240; GFX12-TRUE16-NEXT: v_add_f16_e32 v6.l, v6.l, v5.l4241; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v64242; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4243; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v64244; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v64245; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14246; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=24247; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v04248; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v14249; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v24250; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v34251; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)4252; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)4253; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]4254; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]4255; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s04256; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)4257; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s04258; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x04259; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN4260; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s04261; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_44262; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14263; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s24264; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x04265; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v74266; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v84267; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV4268; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s14269; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)4270; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s14271; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_34272; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end4273; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s14274; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v84275; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x04276; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]4277;4278; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4279; GFX12-FAKE16: ; %bb.0:4280; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x04281; GFX12-FAKE16-NEXT: s_wait_expcnt 0x04282; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x04283; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x04284; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x04285; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v44286; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo4287; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)4288; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v64289; GFX12-FAKE16-NEXT: v_and_b32_e32 v10, -4, v64290; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v44291; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4292; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff4293; GFX12-FAKE16-NEXT: v_not_b32_e32 v11, v74294; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14295; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v04296; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v14297; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v24298; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v34299; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)4300; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)4301; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]4302; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]4303; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s04304; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)4305; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s04306; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x04307; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen4308; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s04309; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_14310; GFX12-FAKE16-NEXT: ; %bb.2:4311; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s14312; GFX12-FAKE16-NEXT: s_mov_b32 s1, 04313; GFX12-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start4314; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=14315; GFX12-FAKE16-NEXT: ; Child Loop BB15_4 Depth 24316; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x04317; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v6, v4, v74318; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo4319; GFX12-FAKE16-NEXT: s_wait_storecnt 0x04320; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4321; GFX12-FAKE16-NEXT: v_add_f16_e32 v6, v6, v54322; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v64323; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4324; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v6, v4, v64325; GFX12-FAKE16-NEXT: v_and_or_b32 v6, v7, v11, v64326; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4327; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v64328; GFX12-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14329; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=24330; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v04331; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v14332; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v24333; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v34334; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)4335; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)4336; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]4337; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]4338; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s04339; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)4340; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s04341; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x04342; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN4343; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s04344; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_44345; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14346; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s24347; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x04348; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v74349; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v84350; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV4351; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s14352; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)4353; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s14354; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_34355; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end4356; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s14357; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v4, v84358; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x04359; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]4360;4361; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4362; GFX942: ; %bb.0:4363; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4364; GFX942-NEXT: v_add_u32_e32 v4, 0x200, v44365; GFX942-NEXT: v_and_b32_e32 v10, -4, v44366; GFX942-NEXT: v_and_b32_e32 v4, 3, v44367; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v44368; GFX942-NEXT: s_mov_b32 s0, 0xffff4369; GFX942-NEXT: v_lshlrev_b32_e64 v6, v4, s04370; GFX942-NEXT: v_not_b32_e32 v11, v64371; GFX942-NEXT: s_mov_b64 s[2:3], exec4372; GFX942-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14373; GFX942-NEXT: v_readfirstlane_b32 s4, v04374; GFX942-NEXT: v_readfirstlane_b32 s5, v14375; GFX942-NEXT: v_readfirstlane_b32 s6, v24376; GFX942-NEXT: v_readfirstlane_b32 s7, v34377; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]4378; GFX942-NEXT: s_nop 04379; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]4380; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]4381; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]4382; GFX942-NEXT: buffer_load_dword v7, v10, s[4:7], 0 offen4383; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]4384; GFX942-NEXT: s_cbranch_execnz .LBB15_14385; GFX942-NEXT: ; %bb.2:4386; GFX942-NEXT: s_mov_b64 exec, s[2:3]4387; GFX942-NEXT: s_mov_b64 s[2:3], 04388; GFX942-NEXT: .LBB15_3: ; %atomicrmw.start4389; GFX942-NEXT: ; =>This Loop Header: Depth=14390; GFX942-NEXT: ; Child Loop BB15_4 Depth 24391; GFX942-NEXT: s_waitcnt vmcnt(0)4392; GFX942-NEXT: v_lshrrev_b32_e32 v6, v4, v74393; GFX942-NEXT: v_add_f16_e32 v6, v6, v54394; GFX942-NEXT: v_lshlrev_b32_e32 v6, v4, v64395; GFX942-NEXT: v_and_or_b32 v6, v7, v11, v64396; GFX942-NEXT: s_mov_b64 s[8:9], exec4397; GFX942-NEXT: v_mov_b64_e32 v[8:9], v[6:7]4398; GFX942-NEXT: buffer_wbl2 sc14399; GFX942-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14400; GFX942-NEXT: ; => This Inner Loop Header: Depth=24401; GFX942-NEXT: v_readfirstlane_b32 s4, v04402; GFX942-NEXT: v_readfirstlane_b32 s5, v14403; GFX942-NEXT: v_readfirstlane_b32 s6, v24404; GFX942-NEXT: v_readfirstlane_b32 s7, v34405; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]4406; GFX942-NEXT: s_nop 04407; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]4408; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]4409; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]4410; GFX942-NEXT: s_waitcnt vmcnt(0)4411; GFX942-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen sc04412; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]4413; GFX942-NEXT: s_cbranch_execnz .LBB15_44414; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14415; GFX942-NEXT: s_mov_b64 exec, s[8:9]4416; GFX942-NEXT: s_waitcnt vmcnt(0)4417; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v8, v74418; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]4419; GFX942-NEXT: v_mov_b32_e32 v7, v84420; GFX942-NEXT: buffer_inv sc14421; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]4422; GFX942-NEXT: s_cbranch_execnz .LBB15_34423; GFX942-NEXT: ; %bb.6: ; %atomicrmw.end4424; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]4425; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v84426; GFX942-NEXT: s_setpc_b64 s[30:31]4427;4428; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4429; GFX11-TRUE16: ; %bb.0:4430; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4431; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v44432; GFX11-TRUE16-NEXT: s_mov_b32 s1, 04433; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo4434; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)4435; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v64436; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v64437; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v44438; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4439; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff4440; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v74441; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14442; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v04443; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v14444; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v24445; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v34446; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)4447; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]4448; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]4449; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s04450; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4451; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s04452; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen4453; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s04454; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_14455; GFX11-TRUE16-NEXT: ; %bb.2:4456; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s24457; GFX11-TRUE16-NEXT: .p2align 64458; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start4459; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=14460; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 24461; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)4462; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v74463; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, 04464; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo4465; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x04466; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)4467; GFX11-TRUE16-NEXT: v_add_f16_e32 v6.l, v6.l, v5.l4468; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v64469; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4470; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v64471; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v64472; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14473; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=24474; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v04475; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v14476; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v24477; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v34478; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)4479; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]4480; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]4481; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s04482; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4483; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s04484; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)4485; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc4486; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s04487; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_44488; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14489; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s24490; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)4491; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v74492; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v84493; GFX11-TRUE16-NEXT: buffer_gl1_inv4494; GFX11-TRUE16-NEXT: buffer_gl0_inv4495; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s14496; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4497; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s14498; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_34499; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end4500; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s14501; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v84502; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]4503;4504; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4505; GFX11-FAKE16: ; %bb.0:4506; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4507; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v44508; GFX11-FAKE16-NEXT: s_mov_b32 s1, 04509; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo4510; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)4511; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v64512; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, -4, v64513; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v44514; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4515; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff4516; GFX11-FAKE16-NEXT: v_not_b32_e32 v11, v74517; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14518; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v04519; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v14520; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v24521; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v34522; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)4523; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]4524; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]4525; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s04526; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4527; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s04528; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen4529; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s04530; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_14531; GFX11-FAKE16-NEXT: ; %bb.2:4532; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s24533; GFX11-FAKE16-NEXT: .p2align 64534; GFX11-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start4535; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=14536; GFX11-FAKE16-NEXT: ; Child Loop BB15_4 Depth 24537; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)4538; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, v4, v74539; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo4540; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x04541; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4542; GFX11-FAKE16-NEXT: v_add_f16_e32 v6, v6, v54543; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v64544; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4545; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, v4, v64546; GFX11-FAKE16-NEXT: v_and_or_b32 v6, v7, v11, v64547; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4548; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v64549; GFX11-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14550; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=24551; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v04552; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v14553; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v24554; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v34555; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)4556; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]4557; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]4558; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s04559; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4560; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s04561; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)4562; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc4563; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s04564; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_44565; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14566; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s24567; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)4568; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v74569; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v84570; GFX11-FAKE16-NEXT: buffer_gl1_inv4571; GFX11-FAKE16-NEXT: buffer_gl0_inv4572; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s14573; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4574; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s14575; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_34576; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end4577; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s14578; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v4, v84579; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]4580;4581; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4582; GFX10: ; %bb.0:4583; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4584; GFX10-NEXT: v_add_nc_u32_e32 v6, 0x200, v44585; GFX10-NEXT: s_mov_b32 s5, 04586; GFX10-NEXT: s_mov_b32 s6, exec_lo4587; GFX10-NEXT: v_and_b32_e32 v4, 3, v64588; GFX10-NEXT: v_and_b32_e32 v10, -4, v64589; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v44590; GFX10-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff4591; GFX10-NEXT: v_not_b32_e32 v11, v74592; GFX10-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14593; GFX10-NEXT: v_readfirstlane_b32 s8, v04594; GFX10-NEXT: v_readfirstlane_b32 s9, v14595; GFX10-NEXT: v_readfirstlane_b32 s10, v24596; GFX10-NEXT: v_readfirstlane_b32 s11, v34597; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]4598; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]4599; GFX10-NEXT: s_and_b32 s4, vcc_lo, s44600; GFX10-NEXT: s_and_saveexec_b32 s4, s44601; GFX10-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen4602; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)4603; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s44604; GFX10-NEXT: s_cbranch_execnz .LBB15_14605; GFX10-NEXT: ; %bb.2:4606; GFX10-NEXT: s_mov_b32 exec_lo, s64607; GFX10-NEXT: .LBB15_3: ; %atomicrmw.start4608; GFX10-NEXT: ; =>This Loop Header: Depth=14609; GFX10-NEXT: ; Child Loop BB15_4 Depth 24610; GFX10-NEXT: s_waitcnt vmcnt(0)4611; GFX10-NEXT: v_lshrrev_b32_e32 v6, v4, v74612; GFX10-NEXT: s_mov_b32 s6, exec_lo4613; GFX10-NEXT: s_waitcnt_vscnt null, 0x04614; GFX10-NEXT: v_add_f16_e32 v6, v6, v54615; GFX10-NEXT: v_lshlrev_b32_sdwa v6, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_04616; GFX10-NEXT: v_and_or_b32 v6, v7, v11, v64617; GFX10-NEXT: v_mov_b32_e32 v9, v74618; GFX10-NEXT: v_mov_b32_e32 v8, v64619; GFX10-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14620; GFX10-NEXT: ; => This Inner Loop Header: Depth=24621; GFX10-NEXT: v_readfirstlane_b32 s8, v04622; GFX10-NEXT: v_readfirstlane_b32 s9, v14623; GFX10-NEXT: v_readfirstlane_b32 s10, v24624; GFX10-NEXT: v_readfirstlane_b32 s11, v34625; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]4626; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]4627; GFX10-NEXT: s_and_b32 s4, vcc_lo, s44628; GFX10-NEXT: s_and_saveexec_b32 s4, s44629; GFX10-NEXT: s_waitcnt vmcnt(0)4630; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc4631; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)4632; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s44633; GFX10-NEXT: s_cbranch_execnz .LBB15_44634; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14635; GFX10-NEXT: s_mov_b32 exec_lo, s64636; GFX10-NEXT: s_waitcnt vmcnt(0)4637; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v74638; GFX10-NEXT: v_mov_b32_e32 v7, v84639; GFX10-NEXT: buffer_gl1_inv4640; GFX10-NEXT: buffer_gl0_inv4641; GFX10-NEXT: s_or_b32 s5, vcc_lo, s54642; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)4643; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s54644; GFX10-NEXT: s_cbranch_execnz .LBB15_34645; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end4646; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s54647; GFX10-NEXT: v_lshrrev_b32_e32 v0, v4, v84648; GFX10-NEXT: s_setpc_b64 s[30:31]4649;4650; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4651; GFX90A: ; %bb.0:4652; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4653; GFX90A-NEXT: v_add_u32_e32 v4, 0x200, v44654; GFX90A-NEXT: v_and_b32_e32 v10, -4, v44655; GFX90A-NEXT: v_and_b32_e32 v4, 3, v44656; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v44657; GFX90A-NEXT: s_mov_b32 s4, 0xffff4658; GFX90A-NEXT: v_lshlrev_b32_e64 v6, v4, s44659; GFX90A-NEXT: v_not_b32_e32 v11, v64660; GFX90A-NEXT: s_mov_b64 s[6:7], exec4661; GFX90A-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14662; GFX90A-NEXT: v_readfirstlane_b32 s8, v04663; GFX90A-NEXT: v_readfirstlane_b32 s9, v14664; GFX90A-NEXT: v_readfirstlane_b32 s10, v24665; GFX90A-NEXT: v_readfirstlane_b32 s11, v34666; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]4667; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]4668; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]4669; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]4670; GFX90A-NEXT: s_nop 04671; GFX90A-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen4672; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]4673; GFX90A-NEXT: s_cbranch_execnz .LBB15_14674; GFX90A-NEXT: ; %bb.2:4675; GFX90A-NEXT: s_mov_b64 exec, s[6:7]4676; GFX90A-NEXT: s_mov_b64 s[6:7], 04677; GFX90A-NEXT: .LBB15_3: ; %atomicrmw.start4678; GFX90A-NEXT: ; =>This Loop Header: Depth=14679; GFX90A-NEXT: ; Child Loop BB15_4 Depth 24680; GFX90A-NEXT: s_waitcnt vmcnt(0)4681; GFX90A-NEXT: v_lshrrev_b32_e32 v6, v4, v74682; GFX90A-NEXT: v_add_f16_e32 v6, v6, v54683; GFX90A-NEXT: v_lshlrev_b32_e32 v6, v4, v64684; GFX90A-NEXT: v_and_or_b32 v6, v7, v11, v64685; GFX90A-NEXT: s_mov_b64 s[12:13], exec4686; GFX90A-NEXT: v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1]4687; GFX90A-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14688; GFX90A-NEXT: ; => This Inner Loop Header: Depth=24689; GFX90A-NEXT: v_readfirstlane_b32 s8, v04690; GFX90A-NEXT: v_readfirstlane_b32 s9, v14691; GFX90A-NEXT: v_readfirstlane_b32 s10, v24692; GFX90A-NEXT: v_readfirstlane_b32 s11, v34693; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]4694; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]4695; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]4696; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]4697; GFX90A-NEXT: s_waitcnt vmcnt(0)4698; GFX90A-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc4699; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]4700; GFX90A-NEXT: s_cbranch_execnz .LBB15_44701; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14702; GFX90A-NEXT: s_mov_b64 exec, s[12:13]4703; GFX90A-NEXT: s_waitcnt vmcnt(0)4704; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v8, v74705; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]4706; GFX90A-NEXT: v_mov_b32_e32 v7, v84707; GFX90A-NEXT: buffer_wbinvl14708; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]4709; GFX90A-NEXT: s_cbranch_execnz .LBB15_34710; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end4711; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]4712; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v84713; GFX90A-NEXT: s_setpc_b64 s[30:31]4714;4715; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4716; GFX908: ; %bb.0:4717; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4718; GFX908-NEXT: v_add_u32_e32 v4, 0x200, v44719; GFX908-NEXT: v_and_b32_e32 v10, -4, v44720; GFX908-NEXT: v_and_b32_e32 v4, 3, v44721; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v44722; GFX908-NEXT: s_mov_b32 s4, 0xffff4723; GFX908-NEXT: v_lshlrev_b32_e64 v6, v4, s44724; GFX908-NEXT: v_not_b32_e32 v11, v64725; GFX908-NEXT: s_mov_b64 s[6:7], exec4726; GFX908-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14727; GFX908-NEXT: v_readfirstlane_b32 s8, v04728; GFX908-NEXT: v_readfirstlane_b32 s9, v14729; GFX908-NEXT: v_readfirstlane_b32 s10, v24730; GFX908-NEXT: v_readfirstlane_b32 s11, v34731; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]4732; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]4733; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]4734; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]4735; GFX908-NEXT: s_nop 04736; GFX908-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen4737; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]4738; GFX908-NEXT: s_cbranch_execnz .LBB15_14739; GFX908-NEXT: ; %bb.2:4740; GFX908-NEXT: s_mov_b64 exec, s[6:7]4741; GFX908-NEXT: s_mov_b64 s[6:7], 04742; GFX908-NEXT: .LBB15_3: ; %atomicrmw.start4743; GFX908-NEXT: ; =>This Loop Header: Depth=14744; GFX908-NEXT: ; Child Loop BB15_4 Depth 24745; GFX908-NEXT: s_waitcnt vmcnt(0)4746; GFX908-NEXT: v_lshrrev_b32_e32 v6, v4, v74747; GFX908-NEXT: v_add_f16_e32 v6, v6, v54748; GFX908-NEXT: v_lshlrev_b32_e32 v6, v4, v64749; GFX908-NEXT: v_and_or_b32 v6, v7, v11, v64750; GFX908-NEXT: v_mov_b32_e32 v9, v74751; GFX908-NEXT: s_mov_b64 s[12:13], exec4752; GFX908-NEXT: v_mov_b32_e32 v8, v64753; GFX908-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14754; GFX908-NEXT: ; => This Inner Loop Header: Depth=24755; GFX908-NEXT: v_readfirstlane_b32 s8, v04756; GFX908-NEXT: v_readfirstlane_b32 s9, v14757; GFX908-NEXT: v_readfirstlane_b32 s10, v24758; GFX908-NEXT: v_readfirstlane_b32 s11, v34759; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]4760; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]4761; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]4762; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]4763; GFX908-NEXT: s_waitcnt vmcnt(0)4764; GFX908-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc4765; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]4766; GFX908-NEXT: s_cbranch_execnz .LBB15_44767; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14768; GFX908-NEXT: s_mov_b64 exec, s[12:13]4769; GFX908-NEXT: s_waitcnt vmcnt(0)4770; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v8, v74771; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]4772; GFX908-NEXT: v_mov_b32_e32 v7, v84773; GFX908-NEXT: buffer_wbinvl14774; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]4775; GFX908-NEXT: s_cbranch_execnz .LBB15_34776; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end4777; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]4778; GFX908-NEXT: v_lshrrev_b32_e32 v0, v4, v84779; GFX908-NEXT: s_setpc_b64 s[30:31]4780;4781; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4782; GFX8: ; %bb.0:4783; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4784; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x200, v44785; GFX8-NEXT: v_and_b32_e32 v10, -4, v44786; GFX8-NEXT: v_and_b32_e32 v4, 3, v44787; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v44788; GFX8-NEXT: s_mov_b32 s4, 0xffff4789; GFX8-NEXT: v_lshlrev_b32_e64 v6, v4, s44790; GFX8-NEXT: v_not_b32_e32 v11, v64791; GFX8-NEXT: s_mov_b64 s[6:7], exec4792; GFX8-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14793; GFX8-NEXT: v_readfirstlane_b32 s8, v04794; GFX8-NEXT: v_readfirstlane_b32 s9, v14795; GFX8-NEXT: v_readfirstlane_b32 s10, v24796; GFX8-NEXT: v_readfirstlane_b32 s11, v34797; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]4798; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]4799; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]4800; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]4801; GFX8-NEXT: s_nop 04802; GFX8-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen4803; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]4804; GFX8-NEXT: s_cbranch_execnz .LBB15_14805; GFX8-NEXT: ; %bb.2:4806; GFX8-NEXT: s_mov_b64 exec, s[6:7]4807; GFX8-NEXT: s_mov_b64 s[6:7], 04808; GFX8-NEXT: .LBB15_3: ; %atomicrmw.start4809; GFX8-NEXT: ; =>This Loop Header: Depth=14810; GFX8-NEXT: ; Child Loop BB15_4 Depth 24811; GFX8-NEXT: s_waitcnt vmcnt(0)4812; GFX8-NEXT: v_lshrrev_b32_e32 v6, v4, v74813; GFX8-NEXT: v_add_f16_e32 v6, v6, v54814; GFX8-NEXT: v_lshlrev_b32_e32 v6, v4, v64815; GFX8-NEXT: v_and_b32_e32 v8, v7, v114816; GFX8-NEXT: v_or_b32_e32 v6, v8, v64817; GFX8-NEXT: v_mov_b32_e32 v9, v74818; GFX8-NEXT: s_mov_b64 s[12:13], exec4819; GFX8-NEXT: v_mov_b32_e32 v8, v64820; GFX8-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14821; GFX8-NEXT: ; => This Inner Loop Header: Depth=24822; GFX8-NEXT: v_readfirstlane_b32 s8, v04823; GFX8-NEXT: v_readfirstlane_b32 s9, v14824; GFX8-NEXT: v_readfirstlane_b32 s10, v24825; GFX8-NEXT: v_readfirstlane_b32 s11, v34826; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]4827; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]4828; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]4829; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]4830; GFX8-NEXT: s_waitcnt vmcnt(0)4831; GFX8-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc4832; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]4833; GFX8-NEXT: s_cbranch_execnz .LBB15_44834; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14835; GFX8-NEXT: s_mov_b64 exec, s[12:13]4836; GFX8-NEXT: s_waitcnt vmcnt(0)4837; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v8, v74838; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]4839; GFX8-NEXT: v_mov_b32_e32 v7, v84840; GFX8-NEXT: buffer_wbinvl14841; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]4842; GFX8-NEXT: s_cbranch_execnz .LBB15_34843; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end4844; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]4845; GFX8-NEXT: v_lshrrev_b32_e32 v0, v4, v84846; GFX8-NEXT: s_setpc_b64 s[30:31]4847;4848; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4849; GFX7: ; %bb.0:4850; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4851; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v44852; GFX7-NEXT: v_and_b32_e32 v8, -4, v44853; GFX7-NEXT: v_and_b32_e32 v4, 3, v44854; GFX7-NEXT: v_lshlrev_b32_e32 v7, 3, v44855; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v74856; GFX7-NEXT: v_not_b32_e32 v9, v44857; GFX7-NEXT: s_mov_b64 s[6:7], exec4858; GFX7-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14859; GFX7-NEXT: v_readfirstlane_b32 s8, v04860; GFX7-NEXT: v_readfirstlane_b32 s9, v14861; GFX7-NEXT: v_readfirstlane_b32 s10, v24862; GFX7-NEXT: v_readfirstlane_b32 s11, v34863; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]4864; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]4865; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]4866; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]4867; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen4868; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]4869; GFX7-NEXT: s_cbranch_execnz .LBB15_14870; GFX7-NEXT: ; %bb.2:4871; GFX7-NEXT: s_mov_b64 exec, s[6:7]4872; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v54873; GFX7-NEXT: s_mov_b64 s[6:7], 04874; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v44875; GFX7-NEXT: .LBB15_3: ; %atomicrmw.start4876; GFX7-NEXT: ; =>This Loop Header: Depth=14877; GFX7-NEXT: ; Child Loop BB15_4 Depth 24878; GFX7-NEXT: s_waitcnt vmcnt(0)4879; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v64880; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v44881; GFX7-NEXT: v_and_b32_e32 v5, v6, v94882; GFX7-NEXT: s_mov_b64 s[12:13], exec4883; GFX7-NEXT: v_add_f32_e32 v4, v4, v104884; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v44885; GFX7-NEXT: v_lshlrev_b32_e32 v4, v7, v44886; GFX7-NEXT: v_or_b32_e32 v5, v5, v44887; GFX7-NEXT: v_mov_b32_e32 v4, v54888; GFX7-NEXT: v_mov_b32_e32 v5, v64889; GFX7-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14890; GFX7-NEXT: ; => This Inner Loop Header: Depth=24891; GFX7-NEXT: v_readfirstlane_b32 s8, v04892; GFX7-NEXT: v_readfirstlane_b32 s9, v14893; GFX7-NEXT: v_readfirstlane_b32 s10, v24894; GFX7-NEXT: v_readfirstlane_b32 s11, v34895; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]4896; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]4897; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]4898; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]4899; GFX7-NEXT: s_waitcnt vmcnt(0)4900; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc4901; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]4902; GFX7-NEXT: s_cbranch_execnz .LBB15_44903; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14904; GFX7-NEXT: s_mov_b64 exec, s[12:13]4905; GFX7-NEXT: s_waitcnt vmcnt(0)4906; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v64907; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]4908; GFX7-NEXT: v_mov_b32_e32 v6, v44909; GFX7-NEXT: buffer_wbinvl14910; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]4911; GFX7-NEXT: s_cbranch_execnz .LBB15_34912; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end4913; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]4914; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v44915; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v04916; GFX7-NEXT: s_setpc_b64 s[30:31]4917;4918; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:4919; GFX6: ; %bb.0:4920; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4921; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v44922; GFX6-NEXT: v_and_b32_e32 v8, -4, v44923; GFX6-NEXT: v_and_b32_e32 v4, 3, v44924; GFX6-NEXT: v_lshlrev_b32_e32 v7, 3, v44925; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v74926; GFX6-NEXT: v_not_b32_e32 v9, v44927; GFX6-NEXT: s_mov_b64 s[6:7], exec4928; GFX6-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=14929; GFX6-NEXT: v_readfirstlane_b32 s8, v04930; GFX6-NEXT: v_readfirstlane_b32 s9, v14931; GFX6-NEXT: v_readfirstlane_b32 s10, v24932; GFX6-NEXT: v_readfirstlane_b32 s11, v34933; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]4934; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]4935; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]4936; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]4937; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen4938; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]4939; GFX6-NEXT: s_cbranch_execnz .LBB15_14940; GFX6-NEXT: ; %bb.2:4941; GFX6-NEXT: s_mov_b64 exec, s[6:7]4942; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v54943; GFX6-NEXT: s_mov_b64 s[6:7], 04944; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v44945; GFX6-NEXT: .LBB15_3: ; %atomicrmw.start4946; GFX6-NEXT: ; =>This Loop Header: Depth=14947; GFX6-NEXT: ; Child Loop BB15_4 Depth 24948; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)4949; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v64950; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v44951; GFX6-NEXT: v_and_b32_e32 v5, v6, v94952; GFX6-NEXT: s_mov_b64 s[12:13], exec4953; GFX6-NEXT: v_add_f32_e32 v4, v4, v104954; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v44955; GFX6-NEXT: v_lshlrev_b32_e32 v4, v7, v44956; GFX6-NEXT: v_or_b32_e32 v5, v5, v44957; GFX6-NEXT: v_mov_b32_e32 v4, v54958; GFX6-NEXT: v_mov_b32_e32 v5, v64959; GFX6-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=14960; GFX6-NEXT: ; => This Inner Loop Header: Depth=24961; GFX6-NEXT: v_readfirstlane_b32 s8, v04962; GFX6-NEXT: v_readfirstlane_b32 s9, v14963; GFX6-NEXT: v_readfirstlane_b32 s10, v24964; GFX6-NEXT: v_readfirstlane_b32 s11, v34965; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]4966; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]4967; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]4968; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]4969; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)4970; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc4971; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]4972; GFX6-NEXT: s_cbranch_execnz .LBB15_44973; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=14974; GFX6-NEXT: s_mov_b64 exec, s[12:13]4975; GFX6-NEXT: s_waitcnt vmcnt(0)4976; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v64977; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]4978; GFX6-NEXT: v_mov_b32_e32 v6, v44979; GFX6-NEXT: buffer_wbinvl14980; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]4981; GFX6-NEXT: s_cbranch_execnz .LBB15_34982; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end4983; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]4984; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v44985; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v04986; GFX6-NEXT: s_waitcnt expcnt(0)4987; GFX6-NEXT: s_setpc_b64 s[30:31]4988 %gep = getelementptr half, ptr addrspace(7) %ptr, i32 2564989 %result = atomicrmw fadd ptr addrspace(7) %gep, half %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !04990 ret half %result4991}4992 4993; --------------------------------------------------------------------4994; bfloat4995; --------------------------------------------------------------------4996 4997define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {4998; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:4999; GFX12-TRUE16: ; %bb.0:5000; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x05001; GFX12-TRUE16-NEXT: s_wait_expcnt 0x05002; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x05003; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x05004; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x05005; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x2005006; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5007; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -45008; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5009; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s45010; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 35011; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5012; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 35013; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5014; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s45015; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen5016; GFX12-TRUE16-NEXT: s_not_b32 s6, s55017; GFX12-TRUE16-NEXT: s_mov_b32 s5, 05018; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start5019; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=15020; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x05021; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v25022; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 05023; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l5024; GFX12-TRUE16-NEXT: s_wait_storecnt 0x05025; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)5026; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v15027; GFX12-TRUE16-NEXT: v_add_f32_e32 v1, v1, v35028; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l5029; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)5030; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 15031; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v15032; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v15033; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff5034; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)5035; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5036; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo5037; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h5038; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)5039; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v35040; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5041; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v15042; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5043; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v15044; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN5045; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x05046; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV5047; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v25048; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v35049; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s55050; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5051; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s55052; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_15053; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end5054; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s55055; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v35056; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x05057; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]5058;5059; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:5060; GFX12-FAKE16: ; %bb.0:5061; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x05062; GFX12-FAKE16-NEXT: s_wait_expcnt 0x05063; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x05064; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x05065; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x05066; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x2005067; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v05068; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5069; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -45070; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5071; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s45072; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 35073; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5074; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 35075; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5076; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s45077; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen5078; GFX12-FAKE16-NEXT: s_not_b32 s6, s55079; GFX12-FAKE16-NEXT: s_mov_b32 s5, 05080; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start5081; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=15082; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x05083; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v15084; GFX12-FAKE16-NEXT: s_wait_storecnt 0x05085; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5086; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v05087; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v55088; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)5089; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 15090; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v05091; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v05092; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff5093; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)5094; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5095; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo5096; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v05097; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)5098; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v05099; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5100; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v05101; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5102; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v05103; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN5104; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x05105; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV5106; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v15107; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v25108; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s55109; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5110; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s55111; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB16_15112; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end5113; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s55114; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v25115; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x05116; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]5117;5118; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:5119; GFX942: ; %bb.0:5120; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5121; GFX942-NEXT: s_addk_i32 s16, 0x2005122; GFX942-NEXT: s_and_b32 s4, s16, -45123; GFX942-NEXT: v_mov_b32_e32 v4, s45124; GFX942-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen5125; GFX942-NEXT: s_and_b32 s4, s16, 35126; GFX942-NEXT: s_lshl_b32 s6, s4, 35127; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s65128; GFX942-NEXT: s_not_b32 s7, s45129; GFX942-NEXT: s_mov_b64 s[4:5], 05130; GFX942-NEXT: v_lshlrev_b32_e32 v5, 16, v05131; GFX942-NEXT: s_movk_i32 s8, 0x7fff5132; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start5133; GFX942-NEXT: ; =>This Inner Loop Header: Depth=15134; GFX942-NEXT: s_waitcnt vmcnt(0)5135; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5136; GFX942-NEXT: buffer_wbl2 sc15137; GFX942-NEXT: v_add_f32_e32 v0, v0, v55138; GFX942-NEXT: v_bfe_u32 v2, v0, 16, 15139; GFX942-NEXT: v_or_b32_e32 v3, 0x400000, v05140; GFX942-NEXT: v_add3_u32 v2, v2, v0, s85141; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v05142; GFX942-NEXT: s_nop 15143; GFX942-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc5144; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15145; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v05146; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1]5147; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[0:3], 0 offen sc05148; GFX942-NEXT: s_waitcnt vmcnt(0)5149; GFX942-NEXT: buffer_inv sc15150; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15151; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5152; GFX942-NEXT: v_mov_b32_e32 v1, v25153; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]5154; GFX942-NEXT: s_cbranch_execnz .LBB16_15155; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end5156; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]5157; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v25158; GFX942-NEXT: s_setpc_b64 s[30:31]5159;5160; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:5161; GFX11-TRUE16: ; %bb.0:5162; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5163; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x2005164; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)5165; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -45166; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s45167; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 35168; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)5169; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 35170; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s45171; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen5172; GFX11-TRUE16-NEXT: s_not_b32 s6, s55173; GFX11-TRUE16-NEXT: s_mov_b32 s5, 05174; GFX11-TRUE16-NEXT: .p2align 65175; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start5176; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=15177; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)5178; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v25179; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 05180; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l5181; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x05182; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)5183; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v15184; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v35185; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l5186; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)5187; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 15188; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v15189; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v15190; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff5191; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5192; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo5193; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h5194; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5195; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v35196; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v15197; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5198; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v15199; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc5200; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)5201; GFX11-TRUE16-NEXT: buffer_gl1_inv5202; GFX11-TRUE16-NEXT: buffer_gl0_inv5203; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v25204; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v35205; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s55206; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5207; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s55208; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_15209; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end5210; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s55211; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v35212; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]5213;5214; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:5215; GFX11-FAKE16: ; %bb.0:5216; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5217; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x2005218; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v05219; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -45220; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5221; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s45222; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 35223; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 35224; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5225; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s45226; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen5227; GFX11-FAKE16-NEXT: s_not_b32 s6, s55228; GFX11-FAKE16-NEXT: s_mov_b32 s5, 05229; GFX11-FAKE16-NEXT: .p2align 65230; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start5231; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=15232; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)5233; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v15234; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x05235; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5236; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v05237; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v55238; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)5239; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 15240; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v05241; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v05242; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff5243; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5244; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo5245; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v05246; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5247; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v05248; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v05249; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5250; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v05251; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc5252; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)5253; GFX11-FAKE16-NEXT: buffer_gl1_inv5254; GFX11-FAKE16-NEXT: buffer_gl0_inv5255; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v15256; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v25257; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s55258; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5259; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s55260; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB16_15261; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end5262; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s55263; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v25264; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]5265;5266; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:5267; GFX10: ; %bb.0:5268; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5269; GFX10-NEXT: s_addk_i32 s20, 0x2005270; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v05271; GFX10-NEXT: s_and_b32 s4, s20, -45272; GFX10-NEXT: v_mov_b32_e32 v4, s45273; GFX10-NEXT: s_and_b32 s4, s20, 35274; GFX10-NEXT: s_lshl_b32 s4, s4, 35275; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s45276; GFX10-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen5277; GFX10-NEXT: s_not_b32 s6, s55278; GFX10-NEXT: s_mov_b32 s5, 05279; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start5280; GFX10-NEXT: ; =>This Inner Loop Header: Depth=15281; GFX10-NEXT: s_waitcnt vmcnt(0)5282; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5283; GFX10-NEXT: s_waitcnt_vscnt null, 0x05284; GFX10-NEXT: v_add_f32_e32 v0, v0, v55285; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 15286; GFX10-NEXT: v_or_b32_e32 v3, 0x400000, v05287; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v05288; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff5289; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo5290; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15291; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v05292; GFX10-NEXT: v_mov_b32_e32 v3, v15293; GFX10-NEXT: v_mov_b32_e32 v2, v05294; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc5295; GFX10-NEXT: s_waitcnt vmcnt(0)5296; GFX10-NEXT: buffer_gl1_inv5297; GFX10-NEXT: buffer_gl0_inv5298; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v15299; GFX10-NEXT: v_mov_b32_e32 v1, v25300; GFX10-NEXT: s_or_b32 s5, vcc_lo, s55301; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s55302; GFX10-NEXT: s_cbranch_execnz .LBB16_15303; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end5304; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s55305; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v25306; GFX10-NEXT: s_setpc_b64 s[30:31]5307;5308; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:5309; GFX90A: ; %bb.0:5310; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5311; GFX90A-NEXT: s_addk_i32 s20, 0x2005312; GFX90A-NEXT: s_and_b32 s4, s20, -45313; GFX90A-NEXT: v_mov_b32_e32 v4, s45314; GFX90A-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen5315; GFX90A-NEXT: s_and_b32 s4, s20, 35316; GFX90A-NEXT: s_lshl_b32 s6, s4, 35317; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s65318; GFX90A-NEXT: s_not_b32 s7, s45319; GFX90A-NEXT: s_mov_b64 s[4:5], 05320; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v05321; GFX90A-NEXT: s_movk_i32 s8, 0x7fff5322; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start5323; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=15324; GFX90A-NEXT: s_waitcnt vmcnt(0)5325; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5326; GFX90A-NEXT: v_add_f32_e32 v0, v0, v55327; GFX90A-NEXT: v_bfe_u32 v2, v0, 16, 15328; GFX90A-NEXT: v_or_b32_e32 v3, 0x400000, v05329; GFX90A-NEXT: v_add3_u32 v2, v2, v0, s85330; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v05331; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc5332; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15333; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v05334; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]5335; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc5336; GFX90A-NEXT: s_waitcnt vmcnt(0)5337; GFX90A-NEXT: buffer_wbinvl15338; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15339; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5340; GFX90A-NEXT: v_mov_b32_e32 v1, v25341; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]5342; GFX90A-NEXT: s_cbranch_execnz .LBB16_15343; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end5344; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5345; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v25346; GFX90A-NEXT: s_setpc_b64 s[30:31]5347;5348; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:5349; GFX908: ; %bb.0:5350; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5351; GFX908-NEXT: s_addk_i32 s20, 0x2005352; GFX908-NEXT: s_and_b32 s4, s20, -45353; GFX908-NEXT: v_mov_b32_e32 v4, s45354; GFX908-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen5355; GFX908-NEXT: s_and_b32 s4, s20, 35356; GFX908-NEXT: s_lshl_b32 s6, s4, 35357; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s65358; GFX908-NEXT: s_not_b32 s7, s45359; GFX908-NEXT: s_mov_b64 s[4:5], 05360; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v05361; GFX908-NEXT: s_movk_i32 s8, 0x7fff5362; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start5363; GFX908-NEXT: ; =>This Inner Loop Header: Depth=15364; GFX908-NEXT: s_waitcnt vmcnt(0)5365; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5366; GFX908-NEXT: v_add_f32_e32 v0, v0, v55367; GFX908-NEXT: v_bfe_u32 v2, v0, 16, 15368; GFX908-NEXT: v_or_b32_e32 v3, 0x400000, v05369; GFX908-NEXT: v_add3_u32 v2, v2, v0, s85370; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v05371; GFX908-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc5372; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15373; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v05374; GFX908-NEXT: v_mov_b32_e32 v3, v15375; GFX908-NEXT: v_mov_b32_e32 v2, v05376; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc5377; GFX908-NEXT: s_waitcnt vmcnt(0)5378; GFX908-NEXT: buffer_wbinvl15379; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15380; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5381; GFX908-NEXT: v_mov_b32_e32 v1, v25382; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]5383; GFX908-NEXT: s_cbranch_execnz .LBB16_15384; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end5385; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]5386; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v25387; GFX908-NEXT: s_setpc_b64 s[30:31]5388;5389; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:5390; GFX8: ; %bb.0:5391; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5392; GFX8-NEXT: s_addk_i32 s20, 0x2005393; GFX8-NEXT: s_and_b32 s4, s20, -45394; GFX8-NEXT: v_mov_b32_e32 v4, s45395; GFX8-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen5396; GFX8-NEXT: s_and_b32 s4, s20, 35397; GFX8-NEXT: s_lshl_b32 s6, s4, 35398; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s65399; GFX8-NEXT: s_not_b32 s7, s45400; GFX8-NEXT: s_mov_b64 s[4:5], 05401; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v05402; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start5403; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15404; GFX8-NEXT: v_mov_b32_e32 v0, s65405; GFX8-NEXT: s_waitcnt vmcnt(0)5406; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5407; GFX8-NEXT: v_add_f32_e32 v3, v3, v55408; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 15409; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v35410; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v65411; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v35412; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v35413; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc5414; GFX8-NEXT: v_and_b32_e32 v2, s7, v15415; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15416; GFX8-NEXT: v_or_b32_e32 v0, v2, v05417; GFX8-NEXT: v_mov_b32_e32 v3, v15418; GFX8-NEXT: v_mov_b32_e32 v2, v05419; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc5420; GFX8-NEXT: s_waitcnt vmcnt(0)5421; GFX8-NEXT: buffer_wbinvl15422; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15423; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5424; GFX8-NEXT: v_mov_b32_e32 v1, v25425; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]5426; GFX8-NEXT: s_cbranch_execnz .LBB16_15427; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5428; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]5429; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v25430; GFX8-NEXT: s_setpc_b64 s[30:31]5431;5432; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:5433; GFX7: ; %bb.0:5434; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5435; GFX7-NEXT: s_addk_i32 s20, 0x2005436; GFX7-NEXT: s_and_b32 s4, s20, -45437; GFX7-NEXT: v_mov_b32_e32 v4, s45438; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen5439; GFX7-NEXT: s_and_b32 s4, s20, 35440; GFX7-NEXT: s_lshl_b32 s6, s4, 35441; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s65442; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v05443; GFX7-NEXT: s_not_b32 s7, s45444; GFX7-NEXT: s_mov_b64 s[4:5], 05445; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v05446; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start5447; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15448; GFX7-NEXT: s_waitcnt vmcnt(0)5449; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v15450; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v05451; GFX7-NEXT: v_add_f32_e32 v0, v0, v55452; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v05453; GFX7-NEXT: v_and_b32_e32 v2, s7, v15454; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v05455; GFX7-NEXT: v_or_b32_e32 v0, v2, v05456; GFX7-NEXT: v_mov_b32_e32 v3, v15457; GFX7-NEXT: v_mov_b32_e32 v2, v05458; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc5459; GFX7-NEXT: s_waitcnt vmcnt(0)5460; GFX7-NEXT: buffer_wbinvl15461; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15462; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5463; GFX7-NEXT: v_mov_b32_e32 v1, v25464; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]5465; GFX7-NEXT: s_cbranch_execnz .LBB16_15466; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5467; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]5468; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v25469; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v05470; GFX7-NEXT: s_setpc_b64 s[30:31]5471;5472; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:5473; GFX6: ; %bb.0:5474; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5475; GFX6-NEXT: s_addk_i32 s20, 0x2005476; GFX6-NEXT: s_and_b32 s4, s20, -45477; GFX6-NEXT: v_mov_b32_e32 v4, s45478; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen5479; GFX6-NEXT: s_and_b32 s4, s20, 35480; GFX6-NEXT: s_lshl_b32 s6, s4, 35481; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s65482; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v05483; GFX6-NEXT: s_not_b32 s7, s45484; GFX6-NEXT: s_mov_b64 s[4:5], 05485; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v05486; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start5487; GFX6-NEXT: ; =>This Inner Loop Header: Depth=15488; GFX6-NEXT: s_waitcnt vmcnt(0)5489; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v15490; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v05491; GFX6-NEXT: v_add_f32_e32 v0, v0, v55492; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v05493; GFX6-NEXT: s_waitcnt expcnt(0)5494; GFX6-NEXT: v_and_b32_e32 v2, s7, v15495; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v05496; GFX6-NEXT: v_or_b32_e32 v0, v2, v05497; GFX6-NEXT: v_mov_b32_e32 v3, v15498; GFX6-NEXT: v_mov_b32_e32 v2, v05499; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc5500; GFX6-NEXT: s_waitcnt vmcnt(0)5501; GFX6-NEXT: buffer_wbinvl15502; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15503; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5504; GFX6-NEXT: v_mov_b32_e32 v1, v25505; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]5506; GFX6-NEXT: s_cbranch_execnz .LBB16_15507; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end5508; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]5509; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v25510; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v05511; GFX6-NEXT: s_waitcnt expcnt(0)5512; GFX6-NEXT: s_setpc_b64 s[30:31]5513 %gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 2565514 %result = atomicrmw fadd ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !05515 ret bfloat %result5516}5517 5518define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {5519; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5520; GFX12-TRUE16: ; %bb.0:5521; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x05522; GFX12-TRUE16-NEXT: s_wait_expcnt 0x05523; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x05524; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x05525; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x05526; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x2005527; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5528; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -45529; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5530; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s45531; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 35532; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5533; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 35534; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5535; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s45536; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen5537; GFX12-TRUE16-NEXT: s_not_b32 s6, s55538; GFX12-TRUE16-NEXT: s_mov_b32 s5, 05539; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start5540; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=15541; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x05542; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v25543; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 05544; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l5545; GFX12-TRUE16-NEXT: s_wait_storecnt 0x05546; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)5547; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v15548; GFX12-TRUE16-NEXT: v_add_f32_e32 v1, v1, v45549; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l5550; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)5551; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 15552; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v15553; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v15554; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff5555; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)5556; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5557; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo5558; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h5559; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)5560; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v45561; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5562; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v15563; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5564; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v15565; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN5566; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x05567; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV5568; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v25569; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v45570; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s55571; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5572; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s55573; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB17_15574; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end5575; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s55576; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x05577; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]5578;5579; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5580; GFX12-FAKE16: ; %bb.0:5581; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x05582; GFX12-FAKE16-NEXT: s_wait_expcnt 0x05583; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x05584; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x05585; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x05586; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x2005587; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v05588; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5589; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -45590; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5591; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s45592; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 35593; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5594; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 35595; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5596; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s45597; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen5598; GFX12-FAKE16-NEXT: s_not_b32 s6, s55599; GFX12-FAKE16-NEXT: s_mov_b32 s5, 05600; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start5601; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=15602; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x05603; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v15604; GFX12-FAKE16-NEXT: s_wait_storecnt 0x05605; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5606; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v05607; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v35608; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)5609; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 15610; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v05611; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v05612; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff5613; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)5614; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5615; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo5616; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v05617; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)5618; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v05619; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5620; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v05621; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5622; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v05623; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN5624; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x05625; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV5626; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v15627; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v45628; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s55629; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5630; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s55631; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB17_15632; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end5633; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s55634; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x05635; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]5636;5637; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5638; GFX942: ; %bb.0:5639; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5640; GFX942-NEXT: s_addk_i32 s16, 0x2005641; GFX942-NEXT: s_and_b32 s4, s16, -45642; GFX942-NEXT: v_mov_b32_e32 v2, s45643; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen5644; GFX942-NEXT: s_and_b32 s4, s16, 35645; GFX942-NEXT: s_lshl_b32 s6, s4, 35646; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s65647; GFX942-NEXT: s_not_b32 s7, s45648; GFX942-NEXT: s_mov_b64 s[4:5], 05649; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v05650; GFX942-NEXT: s_movk_i32 s8, 0x7fff5651; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start5652; GFX942-NEXT: ; =>This Inner Loop Header: Depth=15653; GFX942-NEXT: s_waitcnt vmcnt(0)5654; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5655; GFX942-NEXT: buffer_wbl2 sc15656; GFX942-NEXT: v_add_f32_e32 v0, v0, v35657; GFX942-NEXT: v_bfe_u32 v4, v0, 16, 15658; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v05659; GFX942-NEXT: v_add3_u32 v4, v4, v0, s85660; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v05661; GFX942-NEXT: s_nop 15662; GFX942-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc5663; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15664; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v05665; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]5666; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[0:3], 0 offen sc05667; GFX942-NEXT: s_waitcnt vmcnt(0)5668; GFX942-NEXT: buffer_inv sc15669; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v15670; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5671; GFX942-NEXT: v_mov_b32_e32 v1, v45672; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]5673; GFX942-NEXT: s_cbranch_execnz .LBB17_15674; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end5675; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]5676; GFX942-NEXT: s_setpc_b64 s[30:31]5677;5678; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5679; GFX11-TRUE16: ; %bb.0:5680; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5681; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x2005682; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)5683; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -45684; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s45685; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 35686; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)5687; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 35688; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s45689; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen5690; GFX11-TRUE16-NEXT: s_not_b32 s6, s55691; GFX11-TRUE16-NEXT: s_mov_b32 s5, 05692; GFX11-TRUE16-NEXT: .p2align 65693; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start5694; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=15695; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)5696; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v25697; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 05698; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l5699; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x05700; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)5701; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v15702; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v45703; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l5704; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)5705; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 15706; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v15707; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v15708; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff5709; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5710; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo5711; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h5712; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5713; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v45714; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v15715; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5716; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v15717; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc5718; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)5719; GFX11-TRUE16-NEXT: buffer_gl1_inv5720; GFX11-TRUE16-NEXT: buffer_gl0_inv5721; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v25722; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v45723; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s55724; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5725; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s55726; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB17_15727; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end5728; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s55729; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]5730;5731; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5732; GFX11-FAKE16: ; %bb.0:5733; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5734; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x2005735; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v05736; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -45737; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)5738; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s45739; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 35740; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 35741; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5742; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s45743; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen5744; GFX11-FAKE16-NEXT: s_not_b32 s6, s55745; GFX11-FAKE16-NEXT: s_mov_b32 s5, 05746; GFX11-FAKE16-NEXT: .p2align 65747; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start5748; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=15749; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)5750; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v15751; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x05752; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5753; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v05754; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v35755; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)5756; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 15757; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v05758; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v05759; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff5760; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5761; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo5762; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v05763; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5764; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v05765; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v05766; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5767; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v05768; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc5769; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)5770; GFX11-FAKE16-NEXT: buffer_gl1_inv5771; GFX11-FAKE16-NEXT: buffer_gl0_inv5772; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v15773; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v45774; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s55775; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5776; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s55777; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB17_15778; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end5779; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s55780; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]5781;5782; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5783; GFX10: ; %bb.0:5784; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5785; GFX10-NEXT: s_addk_i32 s20, 0x2005786; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v05787; GFX10-NEXT: s_and_b32 s4, s20, -45788; GFX10-NEXT: v_mov_b32_e32 v2, s45789; GFX10-NEXT: s_and_b32 s4, s20, 35790; GFX10-NEXT: s_lshl_b32 s4, s4, 35791; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s45792; GFX10-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen5793; GFX10-NEXT: s_not_b32 s6, s55794; GFX10-NEXT: s_mov_b32 s5, 05795; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start5796; GFX10-NEXT: ; =>This Inner Loop Header: Depth=15797; GFX10-NEXT: s_waitcnt vmcnt(0)5798; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5799; GFX10-NEXT: s_waitcnt_vscnt null, 0x05800; GFX10-NEXT: v_add_f32_e32 v0, v0, v35801; GFX10-NEXT: v_bfe_u32 v4, v0, 16, 15802; GFX10-NEXT: v_or_b32_e32 v5, 0x400000, v05803; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v05804; GFX10-NEXT: v_add3_u32 v4, v4, v0, 0x7fff5805; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo5806; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15807; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v05808; GFX10-NEXT: v_mov_b32_e32 v5, v15809; GFX10-NEXT: v_mov_b32_e32 v4, v05810; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc5811; GFX10-NEXT: s_waitcnt vmcnt(0)5812; GFX10-NEXT: buffer_gl1_inv5813; GFX10-NEXT: buffer_gl0_inv5814; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v15815; GFX10-NEXT: v_mov_b32_e32 v1, v45816; GFX10-NEXT: s_or_b32 s5, vcc_lo, s55817; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s55818; GFX10-NEXT: s_cbranch_execnz .LBB17_15819; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end5820; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s55821; GFX10-NEXT: s_setpc_b64 s[30:31]5822;5823; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5824; GFX90A: ; %bb.0:5825; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5826; GFX90A-NEXT: s_addk_i32 s20, 0x2005827; GFX90A-NEXT: s_and_b32 s4, s20, -45828; GFX90A-NEXT: v_mov_b32_e32 v2, s45829; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen5830; GFX90A-NEXT: s_and_b32 s4, s20, 35831; GFX90A-NEXT: s_lshl_b32 s6, s4, 35832; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s65833; GFX90A-NEXT: s_not_b32 s7, s45834; GFX90A-NEXT: s_mov_b64 s[4:5], 05835; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v05836; GFX90A-NEXT: s_movk_i32 s8, 0x7fff5837; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start5838; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=15839; GFX90A-NEXT: s_waitcnt vmcnt(0)5840; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5841; GFX90A-NEXT: v_add_f32_e32 v0, v0, v35842; GFX90A-NEXT: v_bfe_u32 v4, v0, 16, 15843; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v05844; GFX90A-NEXT: v_add3_u32 v4, v4, v0, s85845; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v05846; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc5847; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15848; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v05849; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]5850; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc5851; GFX90A-NEXT: s_waitcnt vmcnt(0)5852; GFX90A-NEXT: buffer_wbinvl15853; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v15854; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5855; GFX90A-NEXT: v_mov_b32_e32 v1, v45856; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]5857; GFX90A-NEXT: s_cbranch_execnz .LBB17_15858; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end5859; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5860; GFX90A-NEXT: s_setpc_b64 s[30:31]5861;5862; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5863; GFX908: ; %bb.0:5864; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5865; GFX908-NEXT: s_addk_i32 s20, 0x2005866; GFX908-NEXT: s_and_b32 s4, s20, -45867; GFX908-NEXT: v_mov_b32_e32 v2, s45868; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen5869; GFX908-NEXT: s_and_b32 s4, s20, 35870; GFX908-NEXT: s_lshl_b32 s6, s4, 35871; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s65872; GFX908-NEXT: s_not_b32 s7, s45873; GFX908-NEXT: s_mov_b64 s[4:5], 05874; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v05875; GFX908-NEXT: s_movk_i32 s8, 0x7fff5876; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start5877; GFX908-NEXT: ; =>This Inner Loop Header: Depth=15878; GFX908-NEXT: s_waitcnt vmcnt(0)5879; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5880; GFX908-NEXT: v_add_f32_e32 v0, v0, v35881; GFX908-NEXT: v_bfe_u32 v4, v0, 16, 15882; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v05883; GFX908-NEXT: v_add3_u32 v4, v4, v0, s85884; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v05885; GFX908-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc5886; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15887; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v05888; GFX908-NEXT: v_mov_b32_e32 v5, v15889; GFX908-NEXT: v_mov_b32_e32 v4, v05890; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc5891; GFX908-NEXT: s_waitcnt vmcnt(0)5892; GFX908-NEXT: buffer_wbinvl15893; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v15894; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5895; GFX908-NEXT: v_mov_b32_e32 v1, v45896; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]5897; GFX908-NEXT: s_cbranch_execnz .LBB17_15898; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end5899; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]5900; GFX908-NEXT: s_setpc_b64 s[30:31]5901;5902; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5903; GFX8: ; %bb.0:5904; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5905; GFX8-NEXT: s_addk_i32 s20, 0x2005906; GFX8-NEXT: s_and_b32 s4, s20, -45907; GFX8-NEXT: v_mov_b32_e32 v2, s45908; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen5909; GFX8-NEXT: s_and_b32 s4, s20, 35910; GFX8-NEXT: s_lshl_b32 s6, s4, 35911; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s65912; GFX8-NEXT: s_not_b32 s7, s45913; GFX8-NEXT: s_mov_b64 s[4:5], 05914; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v05915; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start5916; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15917; GFX8-NEXT: v_mov_b32_e32 v0, s65918; GFX8-NEXT: s_waitcnt vmcnt(0)5919; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5920; GFX8-NEXT: v_add_f32_e32 v5, v5, v35921; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 15922; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v55923; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v65924; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v55925; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v55926; GFX8-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc5927; GFX8-NEXT: v_and_b32_e32 v4, s7, v15928; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15929; GFX8-NEXT: v_or_b32_e32 v0, v4, v05930; GFX8-NEXT: v_mov_b32_e32 v5, v15931; GFX8-NEXT: v_mov_b32_e32 v4, v05932; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc5933; GFX8-NEXT: s_waitcnt vmcnt(0)5934; GFX8-NEXT: buffer_wbinvl15935; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v15936; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5937; GFX8-NEXT: v_mov_b32_e32 v1, v45938; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]5939; GFX8-NEXT: s_cbranch_execnz .LBB17_15940; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5941; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]5942; GFX8-NEXT: s_setpc_b64 s[30:31]5943;5944; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5945; GFX7: ; %bb.0:5946; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5947; GFX7-NEXT: s_addk_i32 s20, 0x2005948; GFX7-NEXT: s_and_b32 s4, s20, -45949; GFX7-NEXT: v_mov_b32_e32 v2, s45950; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen5951; GFX7-NEXT: s_and_b32 s4, s20, 35952; GFX7-NEXT: s_lshl_b32 s6, s4, 35953; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s65954; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v05955; GFX7-NEXT: s_not_b32 s7, s45956; GFX7-NEXT: s_mov_b64 s[4:5], 05957; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v05958; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start5959; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15960; GFX7-NEXT: s_waitcnt vmcnt(0)5961; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v15962; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v05963; GFX7-NEXT: v_add_f32_e32 v0, v0, v35964; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v05965; GFX7-NEXT: v_and_b32_e32 v4, s7, v15966; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v05967; GFX7-NEXT: v_or_b32_e32 v0, v4, v05968; GFX7-NEXT: v_mov_b32_e32 v5, v15969; GFX7-NEXT: v_mov_b32_e32 v4, v05970; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc5971; GFX7-NEXT: s_waitcnt vmcnt(0)5972; GFX7-NEXT: buffer_wbinvl15973; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v15974; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5975; GFX7-NEXT: v_mov_b32_e32 v1, v45976; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]5977; GFX7-NEXT: s_cbranch_execnz .LBB17_15978; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5979; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]5980; GFX7-NEXT: s_setpc_b64 s[30:31]5981;5982; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:5983; GFX6: ; %bb.0:5984; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5985; GFX6-NEXT: s_addk_i32 s20, 0x2005986; GFX6-NEXT: s_and_b32 s4, s20, -45987; GFX6-NEXT: v_mov_b32_e32 v2, s45988; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen5989; GFX6-NEXT: s_and_b32 s4, s20, 35990; GFX6-NEXT: s_lshl_b32 s6, s4, 35991; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s65992; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v05993; GFX6-NEXT: s_not_b32 s7, s45994; GFX6-NEXT: s_mov_b64 s[4:5], 05995; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v05996; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start5997; GFX6-NEXT: ; =>This Inner Loop Header: Depth=15998; GFX6-NEXT: s_waitcnt vmcnt(0)5999; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v16000; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v06001; GFX6-NEXT: v_add_f32_e32 v0, v0, v36002; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v06003; GFX6-NEXT: s_waitcnt expcnt(0)6004; GFX6-NEXT: v_and_b32_e32 v4, s7, v16005; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v06006; GFX6-NEXT: v_or_b32_e32 v0, v4, v06007; GFX6-NEXT: v_mov_b32_e32 v5, v16008; GFX6-NEXT: v_mov_b32_e32 v4, v06009; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc6010; GFX6-NEXT: s_waitcnt vmcnt(0)6011; GFX6-NEXT: buffer_wbinvl16012; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v16013; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6014; GFX6-NEXT: v_mov_b32_e32 v1, v46015; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]6016; GFX6-NEXT: s_cbranch_execnz .LBB17_16017; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end6018; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]6019; GFX6-NEXT: s_waitcnt expcnt(0)6020; GFX6-NEXT: s_setpc_b64 s[30:31]6021 %gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 2566022 %unused = atomicrmw fadd ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !06023 ret void6024}6025 6026define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, bfloat %val) #0 {6027; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6028; GFX12-TRUE16: ; %bb.0:6029; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x06030; GFX12-TRUE16-NEXT: s_wait_expcnt 0x06031; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x06032; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x06033; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x06034; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v46035; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo6036; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)6037; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v66038; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v66039; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v46040; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6041; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff6042; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v76043; GFX12-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16044; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v06045; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v16046; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v26047; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v36048; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)6049; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)6050; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]6051; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]6052; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s06053; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)6054; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s06055; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x06056; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen6057; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s06058; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_16059; GFX12-TRUE16-NEXT: ; %bb.2:6060; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s16061; GFX12-TRUE16-NEXT: s_mov_b32 s1, 06062; GFX12-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start6063; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=16064; GFX12-TRUE16-NEXT: ; Child Loop BB18_4 Depth 26065; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x06066; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v76067; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 06068; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l6069; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo6070; GFX12-TRUE16-NEXT: s_wait_storecnt 0x06071; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v66072; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6073; GFX12-TRUE16-NEXT: v_add_f32_e32 v6, v6, v86074; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 16075; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v66076; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v66077; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)6078; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff6079; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)6080; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo6081; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)6082; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h6083; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l6084; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v66085; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6086; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v66087; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v66088; GFX12-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16089; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=26090; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v06091; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v16092; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s6, v26093; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s7, v36094; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)6095; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)6096; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]6097; GFX12-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]6098; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s06099; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)6100; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s06101; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x06102; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN6103; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s06104; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_46105; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16106; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s26107; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x06108; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v76109; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v86110; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV6111; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s16112; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)6113; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s16114; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_36115; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end6116; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s16117; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v86118; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x06119; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]6120;6121; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6122; GFX12-FAKE16: ; %bb.0:6123; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x06124; GFX12-FAKE16-NEXT: s_wait_expcnt 0x06125; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x06126; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x06127; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x06128; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v46129; GFX12-FAKE16-NEXT: s_mov_b32 s1, exec_lo6130; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)6131; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v46132; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v46133; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v66134; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6135; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff6136; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v66137; GFX12-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16138; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v06139; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v16140; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v26141; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v36142; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)6143; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)6144; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]6145; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]6146; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s06147; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)6148; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s06149; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x06150; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen6151; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s06152; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_16153; GFX12-FAKE16-NEXT: ; %bb.2:6154; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s16155; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v56156; GFX12-FAKE16-NEXT: s_mov_b32 s1, 06157; GFX12-FAKE16-NEXT: .LBB18_3: ; %atomicrmw.start6158; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=16159; GFX12-FAKE16-NEXT: ; Child Loop BB18_4 Depth 26160; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x06161; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v66162; GFX12-FAKE16-NEXT: s_mov_b32 s2, exec_lo6163; GFX12-FAKE16-NEXT: s_wait_storecnt 0x06164; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6165; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v46166; GFX12-FAKE16-NEXT: v_add_f32_e32 v4, v4, v106167; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)6168; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 16169; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v46170; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v46171; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff6172; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)6173; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6174; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo6175; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v46176; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6177; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v46178; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v46179; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)6180; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v56181; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v66182; GFX12-FAKE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16183; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=26184; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s4, v06185; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s5, v16186; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s6, v26187; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s7, v36188; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)6189; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)6190; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]6191; GFX12-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]6192; GFX12-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s06193; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)6194; GFX12-FAKE16-NEXT: s_and_saveexec_b32 s0, s06195; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x06196; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN6197; GFX12-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s06198; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_46199; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16200; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s26201; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x06202; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v66203; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v46204; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV6205; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s16206; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)6207; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s16208; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_36209; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end6210; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s16211; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v46212; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x06213; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]6214;6215; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6216; GFX942: ; %bb.0:6217; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6218; GFX942-NEXT: v_add_u32_e32 v4, 0x200, v46219; GFX942-NEXT: v_and_b32_e32 v9, -4, v46220; GFX942-NEXT: v_and_b32_e32 v4, 3, v46221; GFX942-NEXT: v_lshlrev_b32_e32 v8, 3, v46222; GFX942-NEXT: s_mov_b32 s0, 0xffff6223; GFX942-NEXT: v_lshlrev_b32_e64 v4, v8, s06224; GFX942-NEXT: v_not_b32_e32 v10, v46225; GFX942-NEXT: s_mov_b64 s[2:3], exec6226; GFX942-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16227; GFX942-NEXT: v_readfirstlane_b32 s4, v06228; GFX942-NEXT: v_readfirstlane_b32 s5, v16229; GFX942-NEXT: v_readfirstlane_b32 s6, v26230; GFX942-NEXT: v_readfirstlane_b32 s7, v36231; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]6232; GFX942-NEXT: s_nop 06233; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]6234; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]6235; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]6236; GFX942-NEXT: buffer_load_dword v7, v9, s[4:7], 0 offen6237; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]6238; GFX942-NEXT: s_cbranch_execnz .LBB18_16239; GFX942-NEXT: ; %bb.2:6240; GFX942-NEXT: s_mov_b64 exec, s[2:3]6241; GFX942-NEXT: s_mov_b64 s[2:3], 06242; GFX942-NEXT: v_lshlrev_b32_e32 v11, 16, v56243; GFX942-NEXT: s_movk_i32 s10, 0x7fff6244; GFX942-NEXT: .LBB18_3: ; %atomicrmw.start6245; GFX942-NEXT: ; =>This Loop Header: Depth=16246; GFX942-NEXT: ; Child Loop BB18_4 Depth 26247; GFX942-NEXT: s_waitcnt vmcnt(0)6248; GFX942-NEXT: v_lshrrev_b32_sdwa v4, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD6249; GFX942-NEXT: s_mov_b64 s[8:9], exec6250; GFX942-NEXT: v_add_f32_e32 v4, v4, v116251; GFX942-NEXT: v_bfe_u32 v5, v4, 16, 16252; GFX942-NEXT: v_add3_u32 v5, v5, v4, s106253; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v46254; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v46255; GFX942-NEXT: buffer_wbl2 sc16256; GFX942-NEXT: s_nop 06257; GFX942-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc6258; GFX942-NEXT: v_lshlrev_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_16259; GFX942-NEXT: v_and_or_b32 v6, v7, v10, v46260; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[6:7]6261; GFX942-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16262; GFX942-NEXT: ; => This Inner Loop Header: Depth=26263; GFX942-NEXT: v_readfirstlane_b32 s4, v06264; GFX942-NEXT: v_readfirstlane_b32 s5, v16265; GFX942-NEXT: v_readfirstlane_b32 s6, v26266; GFX942-NEXT: v_readfirstlane_b32 s7, v36267; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]6268; GFX942-NEXT: s_nop 06269; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]6270; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]6271; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]6272; GFX942-NEXT: s_waitcnt vmcnt(0)6273; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc06274; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]6275; GFX942-NEXT: s_cbranch_execnz .LBB18_46276; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16277; GFX942-NEXT: s_mov_b64 exec, s[8:9]6278; GFX942-NEXT: s_waitcnt vmcnt(0)6279; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v76280; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]6281; GFX942-NEXT: v_mov_b32_e32 v7, v46282; GFX942-NEXT: buffer_inv sc16283; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]6284; GFX942-NEXT: s_cbranch_execnz .LBB18_36285; GFX942-NEXT: ; %bb.6: ; %atomicrmw.end6286; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]6287; GFX942-NEXT: v_lshrrev_b32_e32 v0, v8, v46288; GFX942-NEXT: s_setpc_b64 s[30:31]6289;6290; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6291; GFX11-TRUE16: ; %bb.0:6292; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6293; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v46294; GFX11-TRUE16-NEXT: s_mov_b32 s1, 06295; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo6296; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)6297; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v66298; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v66299; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v46300; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6301; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff6302; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v76303; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16304; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v06305; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v16306; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v26307; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v36308; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)6309; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]6310; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]6311; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s06312; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6313; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s06314; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen6315; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s06316; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_16317; GFX11-TRUE16-NEXT: ; %bb.2:6318; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s26319; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x16320; GFX11-TRUE16-NEXT: .p2align 66321; GFX11-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start6322; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=16323; GFX11-TRUE16-NEXT: ; Child Loop BB18_4 Depth 26324; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)6325; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v76326; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 06327; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l6328; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo6329; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x06330; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v66331; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6332; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v86333; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 16334; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v66335; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v66336; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)6337; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff6338; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo6339; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)6340; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h6341; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l6342; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v66343; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6344; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v66345; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v66346; GFX11-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16347; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=26348; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v06349; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v16350; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v26351; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v36352; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)6353; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]6354; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]6355; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s06356; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6357; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s06358; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)6359; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc6360; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s06361; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_46362; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16363; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s26364; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)6365; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v76366; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v86367; GFX11-TRUE16-NEXT: buffer_gl1_inv6368; GFX11-TRUE16-NEXT: buffer_gl0_inv6369; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s16370; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6371; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s16372; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_36373; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end6374; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x26375; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s16376; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v86377; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]6378;6379; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6380; GFX11-FAKE16: ; %bb.0:6381; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6382; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v46383; GFX11-FAKE16-NEXT: s_mov_b32 s1, 06384; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo6385; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)6386; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v46387; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v46388; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v66389; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6390; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff6391; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v66392; GFX11-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16393; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v06394; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v16395; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v26396; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v36397; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)6398; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]6399; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]6400; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s06401; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6402; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s06403; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen6404; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s06405; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_16406; GFX11-FAKE16-NEXT: ; %bb.2:6407; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s26408; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v56409; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x16410; GFX11-FAKE16-NEXT: .p2align 66411; GFX11-FAKE16-NEXT: .LBB18_3: ; %atomicrmw.start6412; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=16413; GFX11-FAKE16-NEXT: ; Child Loop BB18_4 Depth 26414; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)6415; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v66416; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo6417; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x06418; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6419; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v46420; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, v4, v106421; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)6422; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 16423; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v46424; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v46425; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff6426; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6427; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo6428; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v46429; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6430; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v46431; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v46432; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)6433; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v56434; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v66435; GFX11-FAKE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16436; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=26437; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v06438; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v16439; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v26440; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v36441; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)6442; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]6443; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]6444; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s06445; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6446; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s06447; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)6448; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc6449; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s06450; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_46451; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16452; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s26453; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)6454; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v66455; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v46456; GFX11-FAKE16-NEXT: buffer_gl1_inv6457; GFX11-FAKE16-NEXT: buffer_gl0_inv6458; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s16459; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6460; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s16461; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_36462; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end6463; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x26464; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s16465; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v46466; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]6467;6468; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6469; GFX10: ; %bb.0:6470; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6471; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v46472; GFX10-NEXT: s_mov_b32 s5, 06473; GFX10-NEXT: s_mov_b32 s6, exec_lo6474; GFX10-NEXT: v_and_b32_e32 v6, 3, v46475; GFX10-NEXT: v_and_b32_e32 v8, -4, v46476; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v66477; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff6478; GFX10-NEXT: v_not_b32_e32 v9, v66479; GFX10-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16480; GFX10-NEXT: v_readfirstlane_b32 s8, v06481; GFX10-NEXT: v_readfirstlane_b32 s9, v16482; GFX10-NEXT: v_readfirstlane_b32 s10, v26483; GFX10-NEXT: v_readfirstlane_b32 s11, v36484; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]6485; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]6486; GFX10-NEXT: s_and_b32 s4, vcc_lo, s46487; GFX10-NEXT: s_and_saveexec_b32 s4, s46488; GFX10-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen6489; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)6490; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s46491; GFX10-NEXT: s_cbranch_execnz .LBB18_16492; GFX10-NEXT: ; %bb.2:6493; GFX10-NEXT: s_mov_b32 exec_lo, s66494; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v56495; GFX10-NEXT: .LBB18_3: ; %atomicrmw.start6496; GFX10-NEXT: ; =>This Loop Header: Depth=16497; GFX10-NEXT: ; Child Loop BB18_4 Depth 26498; GFX10-NEXT: s_waitcnt vmcnt(0)6499; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD6500; GFX10-NEXT: s_mov_b32 s6, exec_lo6501; GFX10-NEXT: s_waitcnt_vscnt null, 0x06502; GFX10-NEXT: v_add_f32_e32 v4, v4, v106503; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 16504; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v46505; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v46506; GFX10-NEXT: v_add3_u32 v5, v5, v4, 0x7fff6507; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo6508; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_16509; GFX10-NEXT: v_and_or_b32 v5, v6, v9, v46510; GFX10-NEXT: v_mov_b32_e32 v4, v56511; GFX10-NEXT: v_mov_b32_e32 v5, v66512; GFX10-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16513; GFX10-NEXT: ; => This Inner Loop Header: Depth=26514; GFX10-NEXT: v_readfirstlane_b32 s8, v06515; GFX10-NEXT: v_readfirstlane_b32 s9, v16516; GFX10-NEXT: v_readfirstlane_b32 s10, v26517; GFX10-NEXT: v_readfirstlane_b32 s11, v36518; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]6519; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]6520; GFX10-NEXT: s_and_b32 s4, vcc_lo, s46521; GFX10-NEXT: s_and_saveexec_b32 s4, s46522; GFX10-NEXT: s_waitcnt vmcnt(0)6523; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc6524; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)6525; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s46526; GFX10-NEXT: s_cbranch_execnz .LBB18_46527; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16528; GFX10-NEXT: s_mov_b32 exec_lo, s66529; GFX10-NEXT: s_waitcnt vmcnt(0)6530; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v66531; GFX10-NEXT: v_mov_b32_e32 v6, v46532; GFX10-NEXT: buffer_gl1_inv6533; GFX10-NEXT: buffer_gl0_inv6534; GFX10-NEXT: s_or_b32 s5, vcc_lo, s56535; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)6536; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s56537; GFX10-NEXT: s_cbranch_execnz .LBB18_36538; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end6539; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s56540; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v46541; GFX10-NEXT: s_setpc_b64 s[30:31]6542;6543; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6544; GFX90A: ; %bb.0:6545; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6546; GFX90A-NEXT: v_add_u32_e32 v4, 0x200, v46547; GFX90A-NEXT: v_and_b32_e32 v9, -4, v46548; GFX90A-NEXT: v_and_b32_e32 v4, 3, v46549; GFX90A-NEXT: v_lshlrev_b32_e32 v8, 3, v46550; GFX90A-NEXT: s_mov_b32 s4, 0xffff6551; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v8, s46552; GFX90A-NEXT: v_not_b32_e32 v10, v46553; GFX90A-NEXT: s_mov_b64 s[6:7], exec6554; GFX90A-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16555; GFX90A-NEXT: v_readfirstlane_b32 s8, v06556; GFX90A-NEXT: v_readfirstlane_b32 s9, v16557; GFX90A-NEXT: v_readfirstlane_b32 s10, v26558; GFX90A-NEXT: v_readfirstlane_b32 s11, v36559; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]6560; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]6561; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]6562; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]6563; GFX90A-NEXT: s_nop 06564; GFX90A-NEXT: buffer_load_dword v7, v9, s[8:11], 0 offen6565; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]6566; GFX90A-NEXT: s_cbranch_execnz .LBB18_16567; GFX90A-NEXT: ; %bb.2:6568; GFX90A-NEXT: s_mov_b64 exec, s[6:7]6569; GFX90A-NEXT: s_mov_b64 s[6:7], 06570; GFX90A-NEXT: v_lshlrev_b32_e32 v11, 16, v56571; GFX90A-NEXT: s_movk_i32 s14, 0x7fff6572; GFX90A-NEXT: .LBB18_3: ; %atomicrmw.start6573; GFX90A-NEXT: ; =>This Loop Header: Depth=16574; GFX90A-NEXT: ; Child Loop BB18_4 Depth 26575; GFX90A-NEXT: s_waitcnt vmcnt(0)6576; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD6577; GFX90A-NEXT: v_add_f32_e32 v4, v4, v116578; GFX90A-NEXT: v_bfe_u32 v5, v4, 16, 16579; GFX90A-NEXT: v_add3_u32 v5, v5, v4, s146580; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v46581; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v46582; GFX90A-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc6583; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_16584; GFX90A-NEXT: v_and_or_b32 v6, v7, v10, v46585; GFX90A-NEXT: s_mov_b64 s[12:13], exec6586; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]6587; GFX90A-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16588; GFX90A-NEXT: ; => This Inner Loop Header: Depth=26589; GFX90A-NEXT: v_readfirstlane_b32 s8, v06590; GFX90A-NEXT: v_readfirstlane_b32 s9, v16591; GFX90A-NEXT: v_readfirstlane_b32 s10, v26592; GFX90A-NEXT: v_readfirstlane_b32 s11, v36593; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]6594; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]6595; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]6596; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]6597; GFX90A-NEXT: s_waitcnt vmcnt(0)6598; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc6599; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]6600; GFX90A-NEXT: s_cbranch_execnz .LBB18_46601; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16602; GFX90A-NEXT: s_mov_b64 exec, s[12:13]6603; GFX90A-NEXT: s_waitcnt vmcnt(0)6604; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v76605; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]6606; GFX90A-NEXT: v_mov_b32_e32 v7, v46607; GFX90A-NEXT: buffer_wbinvl16608; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]6609; GFX90A-NEXT: s_cbranch_execnz .LBB18_36610; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end6611; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]6612; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v8, v46613; GFX90A-NEXT: s_setpc_b64 s[30:31]6614;6615; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6616; GFX908: ; %bb.0:6617; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6618; GFX908-NEXT: v_add_u32_e32 v4, 0x200, v46619; GFX908-NEXT: v_and_b32_e32 v8, -4, v46620; GFX908-NEXT: v_and_b32_e32 v4, 3, v46621; GFX908-NEXT: v_lshlrev_b32_e32 v7, 3, v46622; GFX908-NEXT: s_mov_b32 s4, 0xffff6623; GFX908-NEXT: v_lshlrev_b32_e64 v4, v7, s46624; GFX908-NEXT: v_not_b32_e32 v9, v46625; GFX908-NEXT: s_mov_b64 s[6:7], exec6626; GFX908-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16627; GFX908-NEXT: v_readfirstlane_b32 s8, v06628; GFX908-NEXT: v_readfirstlane_b32 s9, v16629; GFX908-NEXT: v_readfirstlane_b32 s10, v26630; GFX908-NEXT: v_readfirstlane_b32 s11, v36631; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]6632; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]6633; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]6634; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]6635; GFX908-NEXT: s_nop 06636; GFX908-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen6637; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]6638; GFX908-NEXT: s_cbranch_execnz .LBB18_16639; GFX908-NEXT: ; %bb.2:6640; GFX908-NEXT: s_mov_b64 exec, s[6:7]6641; GFX908-NEXT: s_mov_b64 s[6:7], 06642; GFX908-NEXT: v_lshlrev_b32_e32 v10, 16, v56643; GFX908-NEXT: s_movk_i32 s14, 0x7fff6644; GFX908-NEXT: .LBB18_3: ; %atomicrmw.start6645; GFX908-NEXT: ; =>This Loop Header: Depth=16646; GFX908-NEXT: ; Child Loop BB18_4 Depth 26647; GFX908-NEXT: s_waitcnt vmcnt(0)6648; GFX908-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD6649; GFX908-NEXT: v_add_f32_e32 v4, v4, v106650; GFX908-NEXT: v_bfe_u32 v5, v4, 16, 16651; GFX908-NEXT: v_add3_u32 v5, v5, v4, s146652; GFX908-NEXT: v_or_b32_e32 v11, 0x400000, v46653; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v46654; GFX908-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc6655; GFX908-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_16656; GFX908-NEXT: v_and_or_b32 v5, v6, v9, v46657; GFX908-NEXT: v_mov_b32_e32 v4, v56658; GFX908-NEXT: s_mov_b64 s[12:13], exec6659; GFX908-NEXT: v_mov_b32_e32 v5, v66660; GFX908-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16661; GFX908-NEXT: ; => This Inner Loop Header: Depth=26662; GFX908-NEXT: v_readfirstlane_b32 s8, v06663; GFX908-NEXT: v_readfirstlane_b32 s9, v16664; GFX908-NEXT: v_readfirstlane_b32 s10, v26665; GFX908-NEXT: v_readfirstlane_b32 s11, v36666; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]6667; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]6668; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]6669; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]6670; GFX908-NEXT: s_waitcnt vmcnt(0)6671; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc6672; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]6673; GFX908-NEXT: s_cbranch_execnz .LBB18_46674; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16675; GFX908-NEXT: s_mov_b64 exec, s[12:13]6676; GFX908-NEXT: s_waitcnt vmcnt(0)6677; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v66678; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]6679; GFX908-NEXT: v_mov_b32_e32 v6, v46680; GFX908-NEXT: buffer_wbinvl16681; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]6682; GFX908-NEXT: s_cbranch_execnz .LBB18_36683; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end6684; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]6685; GFX908-NEXT: v_lshrrev_b32_e32 v0, v7, v46686; GFX908-NEXT: s_setpc_b64 s[30:31]6687;6688; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6689; GFX8: ; %bb.0:6690; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6691; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x200, v46692; GFX8-NEXT: v_and_b32_e32 v8, -4, v46693; GFX8-NEXT: v_and_b32_e32 v4, 3, v46694; GFX8-NEXT: v_lshlrev_b32_e32 v7, 3, v46695; GFX8-NEXT: s_mov_b32 s4, 0xffff6696; GFX8-NEXT: v_lshlrev_b32_e64 v4, v7, s46697; GFX8-NEXT: v_not_b32_e32 v9, v46698; GFX8-NEXT: s_mov_b64 s[6:7], exec6699; GFX8-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16700; GFX8-NEXT: v_readfirstlane_b32 s8, v06701; GFX8-NEXT: v_readfirstlane_b32 s9, v16702; GFX8-NEXT: v_readfirstlane_b32 s10, v26703; GFX8-NEXT: v_readfirstlane_b32 s11, v36704; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]6705; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]6706; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]6707; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]6708; GFX8-NEXT: s_nop 06709; GFX8-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen6710; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]6711; GFX8-NEXT: s_cbranch_execnz .LBB18_16712; GFX8-NEXT: ; %bb.2:6713; GFX8-NEXT: s_mov_b64 exec, s[6:7]6714; GFX8-NEXT: s_mov_b64 s[6:7], 06715; GFX8-NEXT: v_lshlrev_b32_e32 v10, 16, v56716; GFX8-NEXT: .LBB18_3: ; %atomicrmw.start6717; GFX8-NEXT: ; =>This Loop Header: Depth=16718; GFX8-NEXT: ; Child Loop BB18_4 Depth 26719; GFX8-NEXT: s_waitcnt vmcnt(0)6720; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD6721; GFX8-NEXT: v_add_f32_e32 v4, v4, v106722; GFX8-NEXT: v_bfe_u32 v5, v4, 16, 16723; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v46724; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v56725; GFX8-NEXT: v_or_b32_e32 v11, 0x400000, v46726; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v46727; GFX8-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc6728; GFX8-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_16729; GFX8-NEXT: v_and_b32_e32 v5, v6, v96730; GFX8-NEXT: v_or_b32_e32 v5, v5, v46731; GFX8-NEXT: v_mov_b32_e32 v4, v56732; GFX8-NEXT: s_mov_b64 s[12:13], exec6733; GFX8-NEXT: v_mov_b32_e32 v5, v66734; GFX8-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16735; GFX8-NEXT: ; => This Inner Loop Header: Depth=26736; GFX8-NEXT: v_readfirstlane_b32 s8, v06737; GFX8-NEXT: v_readfirstlane_b32 s9, v16738; GFX8-NEXT: v_readfirstlane_b32 s10, v26739; GFX8-NEXT: v_readfirstlane_b32 s11, v36740; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]6741; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]6742; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]6743; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]6744; GFX8-NEXT: s_waitcnt vmcnt(0)6745; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc6746; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]6747; GFX8-NEXT: s_cbranch_execnz .LBB18_46748; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16749; GFX8-NEXT: s_mov_b64 exec, s[12:13]6750; GFX8-NEXT: s_waitcnt vmcnt(0)6751; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v66752; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]6753; GFX8-NEXT: v_mov_b32_e32 v6, v46754; GFX8-NEXT: buffer_wbinvl16755; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]6756; GFX8-NEXT: s_cbranch_execnz .LBB18_36757; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end6758; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]6759; GFX8-NEXT: v_lshrrev_b32_e32 v0, v7, v46760; GFX8-NEXT: s_setpc_b64 s[30:31]6761;6762; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6763; GFX7: ; %bb.0:6764; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6765; GFX7-NEXT: v_add_i32_e32 v4, vcc, 0x200, v46766; GFX7-NEXT: v_and_b32_e32 v8, -4, v46767; GFX7-NEXT: v_and_b32_e32 v4, 3, v46768; GFX7-NEXT: v_lshlrev_b32_e32 v7, 3, v46769; GFX7-NEXT: v_lshl_b32_e32 v4, 0xffff, v76770; GFX7-NEXT: v_not_b32_e32 v9, v46771; GFX7-NEXT: s_mov_b64 s[6:7], exec6772; GFX7-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16773; GFX7-NEXT: v_readfirstlane_b32 s8, v06774; GFX7-NEXT: v_readfirstlane_b32 s9, v16775; GFX7-NEXT: v_readfirstlane_b32 s10, v26776; GFX7-NEXT: v_readfirstlane_b32 s11, v36777; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]6778; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]6779; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]6780; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]6781; GFX7-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen6782; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]6783; GFX7-NEXT: s_cbranch_execnz .LBB18_16784; GFX7-NEXT: ; %bb.2:6785; GFX7-NEXT: s_mov_b64 exec, s[6:7]6786; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v56787; GFX7-NEXT: s_mov_b64 s[6:7], 06788; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v46789; GFX7-NEXT: .LBB18_3: ; %atomicrmw.start6790; GFX7-NEXT: ; =>This Loop Header: Depth=16791; GFX7-NEXT: ; Child Loop BB18_4 Depth 26792; GFX7-NEXT: s_waitcnt vmcnt(0)6793; GFX7-NEXT: v_lshrrev_b32_e32 v4, v7, v66794; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v46795; GFX7-NEXT: v_add_f32_e32 v4, v4, v106796; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v46797; GFX7-NEXT: v_lshlrev_b32_e32 v4, v7, v46798; GFX7-NEXT: v_and_b32_e32 v5, v6, v96799; GFX7-NEXT: v_or_b32_e32 v5, v5, v46800; GFX7-NEXT: v_mov_b32_e32 v4, v56801; GFX7-NEXT: s_mov_b64 s[12:13], exec6802; GFX7-NEXT: v_mov_b32_e32 v5, v66803; GFX7-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16804; GFX7-NEXT: ; => This Inner Loop Header: Depth=26805; GFX7-NEXT: v_readfirstlane_b32 s8, v06806; GFX7-NEXT: v_readfirstlane_b32 s9, v16807; GFX7-NEXT: v_readfirstlane_b32 s10, v26808; GFX7-NEXT: v_readfirstlane_b32 s11, v36809; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]6810; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]6811; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]6812; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]6813; GFX7-NEXT: s_waitcnt vmcnt(0)6814; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc6815; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]6816; GFX7-NEXT: s_cbranch_execnz .LBB18_46817; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16818; GFX7-NEXT: s_mov_b64 exec, s[12:13]6819; GFX7-NEXT: s_waitcnt vmcnt(0)6820; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v66821; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]6822; GFX7-NEXT: v_mov_b32_e32 v6, v46823; GFX7-NEXT: buffer_wbinvl16824; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]6825; GFX7-NEXT: s_cbranch_execnz .LBB18_36826; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end6827; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]6828; GFX7-NEXT: v_lshrrev_b32_e32 v0, v7, v46829; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v06830; GFX7-NEXT: s_setpc_b64 s[30:31]6831;6832; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:6833; GFX6: ; %bb.0:6834; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6835; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x200, v46836; GFX6-NEXT: v_and_b32_e32 v8, -4, v46837; GFX6-NEXT: v_and_b32_e32 v4, 3, v46838; GFX6-NEXT: v_lshlrev_b32_e32 v7, 3, v46839; GFX6-NEXT: v_lshl_b32_e32 v4, 0xffff, v76840; GFX6-NEXT: v_not_b32_e32 v9, v46841; GFX6-NEXT: s_mov_b64 s[6:7], exec6842; GFX6-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=16843; GFX6-NEXT: v_readfirstlane_b32 s8, v06844; GFX6-NEXT: v_readfirstlane_b32 s9, v16845; GFX6-NEXT: v_readfirstlane_b32 s10, v26846; GFX6-NEXT: v_readfirstlane_b32 s11, v36847; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]6848; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]6849; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]6850; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]6851; GFX6-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen6852; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]6853; GFX6-NEXT: s_cbranch_execnz .LBB18_16854; GFX6-NEXT: ; %bb.2:6855; GFX6-NEXT: s_mov_b64 exec, s[6:7]6856; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v56857; GFX6-NEXT: s_mov_b64 s[6:7], 06858; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v46859; GFX6-NEXT: .LBB18_3: ; %atomicrmw.start6860; GFX6-NEXT: ; =>This Loop Header: Depth=16861; GFX6-NEXT: ; Child Loop BB18_4 Depth 26862; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)6863; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v66864; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v46865; GFX6-NEXT: v_add_f32_e32 v4, v4, v106866; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v46867; GFX6-NEXT: v_lshlrev_b32_e32 v4, v7, v46868; GFX6-NEXT: v_and_b32_e32 v5, v6, v96869; GFX6-NEXT: v_or_b32_e32 v5, v5, v46870; GFX6-NEXT: v_mov_b32_e32 v4, v56871; GFX6-NEXT: s_mov_b64 s[12:13], exec6872; GFX6-NEXT: v_mov_b32_e32 v5, v66873; GFX6-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=16874; GFX6-NEXT: ; => This Inner Loop Header: Depth=26875; GFX6-NEXT: v_readfirstlane_b32 s8, v06876; GFX6-NEXT: v_readfirstlane_b32 s9, v16877; GFX6-NEXT: v_readfirstlane_b32 s10, v26878; GFX6-NEXT: v_readfirstlane_b32 s11, v36879; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]6880; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]6881; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]6882; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]6883; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)6884; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc6885; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]6886; GFX6-NEXT: s_cbranch_execnz .LBB18_46887; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=16888; GFX6-NEXT: s_mov_b64 exec, s[12:13]6889; GFX6-NEXT: s_waitcnt vmcnt(0)6890; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v66891; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]6892; GFX6-NEXT: v_mov_b32_e32 v6, v46893; GFX6-NEXT: buffer_wbinvl16894; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]6895; GFX6-NEXT: s_cbranch_execnz .LBB18_36896; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end6897; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]6898; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v46899; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v06900; GFX6-NEXT: s_waitcnt expcnt(0)6901; GFX6-NEXT: s_setpc_b64 s[30:31]6902 %gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 2566903 %result = atomicrmw fadd ptr addrspace(7) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !06904 ret bfloat %result6905}6906 6907; --------------------------------------------------------------------6908; <2 x half>6909; --------------------------------------------------------------------6910 6911define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {6912; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:6913; GFX12: ; %bb.0:6914; GFX12-NEXT: s_wait_loadcnt_dscnt 0x06915; GFX12-NEXT: s_wait_expcnt 0x06916; GFX12-NEXT: s_wait_samplecnt 0x06917; GFX12-NEXT: s_wait_bvhcnt 0x06918; GFX12-NEXT: s_wait_kmcnt 0x06919; GFX12-NEXT: v_mov_b32_e32 v1, s166920; GFX12-NEXT: s_wait_storecnt 0x06921; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN6922; GFX12-NEXT: s_wait_loadcnt 0x06923; GFX12-NEXT: global_inv scope:SCOPE_DEV6924; GFX12-NEXT: s_wait_loadcnt 0x06925; GFX12-NEXT: s_setpc_b64 s[30:31]6926;6927; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:6928; GFX942: ; %bb.0:6929; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6930; GFX942-NEXT: v_mov_b32_e32 v1, s166931; GFX942-NEXT: buffer_wbl2 sc16932; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:1024 sc06933; GFX942-NEXT: s_waitcnt vmcnt(0)6934; GFX942-NEXT: buffer_inv sc16935; GFX942-NEXT: s_setpc_b64 s[30:31]6936;6937; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:6938; GFX11: ; %bb.0:6939; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6940; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s166941; GFX11-NEXT: v_mov_b32_e32 v0, s166942; GFX11-NEXT: s_mov_b32 s4, 06943; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:10246944; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start6945; GFX11-NEXT: ; =>This Inner Loop Header: Depth=16946; GFX11-NEXT: s_waitcnt vmcnt(0)6947; GFX11-NEXT: v_mov_b32_e32 v5, v06948; GFX11-NEXT: s_waitcnt_vscnt null, 0x06949; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6950; GFX11-NEXT: v_pk_add_f16 v4, v5, v26951; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v56952; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc6953; GFX11-NEXT: s_waitcnt vmcnt(0)6954; GFX11-NEXT: buffer_gl1_inv6955; GFX11-NEXT: buffer_gl0_inv6956; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v56957; GFX11-NEXT: s_or_b32 s4, vcc_lo, s46958; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6959; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s46960; GFX11-NEXT: s_cbranch_execnz .LBB19_16961; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end6962; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s46963; GFX11-NEXT: s_setpc_b64 s[30:31]6964;6965; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:6966; GFX10: ; %bb.0:6967; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6968; GFX10-NEXT: v_mov_b32_e32 v2, v06969; GFX10-NEXT: v_mov_b32_e32 v0, s206970; GFX10-NEXT: v_mov_b32_e32 v3, s206971; GFX10-NEXT: s_mov_b32 s4, 06972; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10246973; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start6974; GFX10-NEXT: ; =>This Inner Loop Header: Depth=16975; GFX10-NEXT: s_waitcnt vmcnt(0)6976; GFX10-NEXT: v_mov_b32_e32 v5, v06977; GFX10-NEXT: s_waitcnt_vscnt null, 0x06978; GFX10-NEXT: v_pk_add_f16 v4, v5, v26979; GFX10-NEXT: v_mov_b32_e32 v0, v46980; GFX10-NEXT: v_mov_b32_e32 v1, v56981; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc6982; GFX10-NEXT: s_waitcnt vmcnt(0)6983; GFX10-NEXT: buffer_gl1_inv6984; GFX10-NEXT: buffer_gl0_inv6985; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v56986; GFX10-NEXT: s_or_b32 s4, vcc_lo, s46987; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s46988; GFX10-NEXT: s_cbranch_execnz .LBB19_16989; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end6990; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s46991; GFX10-NEXT: s_setpc_b64 s[30:31]6992;6993; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:6994; GFX90A: ; %bb.0:6995; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6996; GFX90A-NEXT: v_mov_b32_e32 v1, s206997; GFX90A-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[16:19], 0 offen offset:1024 glc6998; GFX90A-NEXT: s_waitcnt vmcnt(0)6999; GFX90A-NEXT: buffer_wbinvl17000; GFX90A-NEXT: s_setpc_b64 s[30:31]7001;7002; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:7003; GFX908: ; %bb.0:7004; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7005; GFX908-NEXT: v_mov_b32_e32 v2, v07006; GFX908-NEXT: v_mov_b32_e32 v0, s207007; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10247008; GFX908-NEXT: s_mov_b64 s[4:5], 07009; GFX908-NEXT: v_mov_b32_e32 v3, s207010; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start7011; GFX908-NEXT: ; =>This Inner Loop Header: Depth=17012; GFX908-NEXT: s_waitcnt vmcnt(0)7013; GFX908-NEXT: v_mov_b32_e32 v5, v07014; GFX908-NEXT: v_pk_add_f16 v4, v5, v27015; GFX908-NEXT: v_mov_b32_e32 v0, v47016; GFX908-NEXT: v_mov_b32_e32 v1, v57017; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc7018; GFX908-NEXT: s_waitcnt vmcnt(0)7019; GFX908-NEXT: buffer_wbinvl17020; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v57021; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7022; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]7023; GFX908-NEXT: s_cbranch_execnz .LBB19_17024; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end7025; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]7026; GFX908-NEXT: s_setpc_b64 s[30:31]7027;7028; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:7029; GFX8: ; %bb.0:7030; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7031; GFX8-NEXT: v_mov_b32_e32 v2, v07032; GFX8-NEXT: v_mov_b32_e32 v0, s207033; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10247034; GFX8-NEXT: s_mov_b64 s[4:5], 07035; GFX8-NEXT: v_mov_b32_e32 v3, s207036; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start7037; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17038; GFX8-NEXT: s_waitcnt vmcnt(0)7039; GFX8-NEXT: v_mov_b32_e32 v5, v07040; GFX8-NEXT: v_add_f16_sdwa v0, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_17041; GFX8-NEXT: v_add_f16_e32 v1, v5, v27042; GFX8-NEXT: v_or_b32_e32 v4, v1, v07043; GFX8-NEXT: v_mov_b32_e32 v0, v47044; GFX8-NEXT: v_mov_b32_e32 v1, v57045; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc7046; GFX8-NEXT: s_waitcnt vmcnt(0)7047; GFX8-NEXT: buffer_wbinvl17048; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v57049; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7050; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]7051; GFX8-NEXT: s_cbranch_execnz .LBB19_17052; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7053; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]7054; GFX8-NEXT: s_setpc_b64 s[30:31]7055;7056; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:7057; GFX7: ; %bb.0:7058; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7059; GFX7-NEXT: v_mov_b32_e32 v2, s207060; GFX7-NEXT: buffer_load_dword v3, v2, s[16:19], 0 offen offset:10247061; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v17062; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v07063; GFX7-NEXT: s_mov_b64 s[4:5], 07064; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v17065; GFX7-NEXT: s_waitcnt vmcnt(0)7066; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v37067; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v37068; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v17069; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v47070; GFX7-NEXT: v_mov_b32_e32 v4, s207071; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start7072; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17073; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v17074; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v07075; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v17076; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v07077; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v17078; GFX7-NEXT: v_add_f32_e32 v5, v5, v27079; GFX7-NEXT: v_add_f32_e32 v6, v6, v37080; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v57081; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v67082; GFX7-NEXT: v_or_b32_e32 v6, v0, v17083; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v57084; GFX7-NEXT: v_or_b32_e32 v5, v7, v07085; GFX7-NEXT: v_mov_b32_e32 v8, v67086; GFX7-NEXT: v_mov_b32_e32 v7, v57087; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[16:19], 0 offen offset:1024 glc7088; GFX7-NEXT: s_waitcnt vmcnt(0)7089; GFX7-NEXT: buffer_wbinvl17090; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v77091; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v77092; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v17093; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v67094; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7095; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]7096; GFX7-NEXT: s_cbranch_execnz .LBB19_17097; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7098; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]7099; GFX7-NEXT: s_setpc_b64 s[30:31]7100;7101; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:7102; GFX6: ; %bb.0:7103; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7104; GFX6-NEXT: v_mov_b32_e32 v2, s207105; GFX6-NEXT: buffer_load_dword v3, v2, s[16:19], 0 offen offset:10247106; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v17107; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v07108; GFX6-NEXT: s_add_i32 s6, s20, 0x4007109; GFX6-NEXT: s_mov_b64 s[4:5], 07110; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v17111; GFX6-NEXT: s_waitcnt vmcnt(0)7112; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v37113; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v37114; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v17115; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v47116; GFX6-NEXT: v_mov_b32_e32 v4, s67117; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start7118; GFX6-NEXT: ; =>This Inner Loop Header: Depth=17119; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v17120; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v07121; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v17122; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v07123; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v17124; GFX6-NEXT: v_add_f32_e32 v5, v5, v27125; GFX6-NEXT: v_add_f32_e32 v6, v6, v37126; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v57127; GFX6-NEXT: s_waitcnt expcnt(0)7128; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v67129; GFX6-NEXT: v_or_b32_e32 v6, v0, v17130; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v57131; GFX6-NEXT: v_or_b32_e32 v5, v7, v07132; GFX6-NEXT: v_mov_b32_e32 v8, v67133; GFX6-NEXT: v_mov_b32_e32 v7, v57134; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[16:19], 0 offen glc7135; GFX6-NEXT: s_waitcnt vmcnt(0)7136; GFX6-NEXT: buffer_wbinvl17137; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v77138; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v77139; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v17140; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v67141; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7142; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]7143; GFX6-NEXT: s_cbranch_execnz .LBB19_17144; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end7145; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]7146; GFX6-NEXT: s_waitcnt expcnt(0)7147; GFX6-NEXT: s_setpc_b64 s[30:31]7148 %gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 2567149 %result = atomicrmw fadd ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !07150 ret <2 x half> %result7151}7152 7153define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {7154; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:7155; GFX12: ; %bb.0:7156; GFX12-NEXT: s_wait_loadcnt_dscnt 0x07157; GFX12-NEXT: s_wait_expcnt 0x07158; GFX12-NEXT: s_wait_samplecnt 0x07159; GFX12-NEXT: s_wait_bvhcnt 0x07160; GFX12-NEXT: s_wait_kmcnt 0x07161; GFX12-NEXT: v_mov_b32_e32 v1, s167162; GFX12-NEXT: s_wait_storecnt 0x07163; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:10247164; GFX12-NEXT: s_wait_storecnt 0x07165; GFX12-NEXT: global_inv scope:SCOPE_DEV7166; GFX12-NEXT: s_wait_loadcnt 0x07167; GFX12-NEXT: s_setpc_b64 s[30:31]7168;7169; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:7170; GFX942: ; %bb.0:7171; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7172; GFX942-NEXT: v_mov_b32_e32 v1, s167173; GFX942-NEXT: buffer_wbl2 sc17174; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:10247175; GFX942-NEXT: s_waitcnt vmcnt(0)7176; GFX942-NEXT: buffer_inv sc17177; GFX942-NEXT: s_setpc_b64 s[30:31]7178;7179; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:7180; GFX11: ; %bb.0:7181; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7182; GFX11-NEXT: v_mov_b32_e32 v1, s167183; GFX11-NEXT: v_mov_b32_e32 v3, s167184; GFX11-NEXT: s_mov_b32 s4, 07185; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen offset:10247186; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start7187; GFX11-NEXT: ; =>This Inner Loop Header: Depth=17188; GFX11-NEXT: s_waitcnt vmcnt(0)7189; GFX11-NEXT: v_pk_add_f16 v1, v2, v07190; GFX11-NEXT: v_mov_b32_e32 v5, v27191; GFX11-NEXT: s_waitcnt_vscnt null, 0x07192; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)7193; GFX11-NEXT: v_mov_b32_e32 v4, v17194; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc7195; GFX11-NEXT: s_waitcnt vmcnt(0)7196; GFX11-NEXT: buffer_gl1_inv7197; GFX11-NEXT: buffer_gl0_inv7198; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v27199; GFX11-NEXT: v_mov_b32_e32 v2, v47200; GFX11-NEXT: s_or_b32 s4, vcc_lo, s47201; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7202; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s47203; GFX11-NEXT: s_cbranch_execnz .LBB20_17204; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end7205; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s47206; GFX11-NEXT: s_setpc_b64 s[30:31]7207;7208; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:7209; GFX10: ; %bb.0:7210; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7211; GFX10-NEXT: v_mov_b32_e32 v1, s207212; GFX10-NEXT: v_mov_b32_e32 v3, s207213; GFX10-NEXT: s_mov_b32 s4, 07214; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10247215; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start7216; GFX10-NEXT: ; =>This Inner Loop Header: Depth=17217; GFX10-NEXT: s_waitcnt vmcnt(0)7218; GFX10-NEXT: v_pk_add_f16 v1, v2, v07219; GFX10-NEXT: v_mov_b32_e32 v5, v27220; GFX10-NEXT: s_waitcnt_vscnt null, 0x07221; GFX10-NEXT: v_mov_b32_e32 v4, v17222; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc7223; GFX10-NEXT: s_waitcnt vmcnt(0)7224; GFX10-NEXT: buffer_gl1_inv7225; GFX10-NEXT: buffer_gl0_inv7226; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v27227; GFX10-NEXT: v_mov_b32_e32 v2, v47228; GFX10-NEXT: s_or_b32 s4, vcc_lo, s47229; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s47230; GFX10-NEXT: s_cbranch_execnz .LBB20_17231; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end7232; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s47233; GFX10-NEXT: s_setpc_b64 s[30:31]7234;7235; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:7236; GFX90A: ; %bb.0:7237; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7238; GFX90A-NEXT: v_mov_b32_e32 v1, s207239; GFX90A-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[16:19], 0 offen offset:10247240; GFX90A-NEXT: s_waitcnt vmcnt(0)7241; GFX90A-NEXT: buffer_wbinvl17242; GFX90A-NEXT: s_setpc_b64 s[30:31]7243;7244; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:7245; GFX908: ; %bb.0:7246; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7247; GFX908-NEXT: v_mov_b32_e32 v1, s207248; GFX908-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[16:19], 0 offen offset:10247249; GFX908-NEXT: s_waitcnt vmcnt(0)7250; GFX908-NEXT: buffer_wbinvl17251; GFX908-NEXT: s_setpc_b64 s[30:31]7252;7253; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:7254; GFX8: ; %bb.0:7255; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7256; GFX8-NEXT: v_mov_b32_e32 v1, s207257; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10247258; GFX8-NEXT: s_mov_b64 s[4:5], 07259; GFX8-NEXT: v_mov_b32_e32 v3, s207260; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start7261; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17262; GFX8-NEXT: s_waitcnt vmcnt(0)7263; GFX8-NEXT: v_add_f16_sdwa v1, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_17264; GFX8-NEXT: v_add_f16_e32 v4, v2, v07265; GFX8-NEXT: v_or_b32_e32 v1, v4, v17266; GFX8-NEXT: v_mov_b32_e32 v5, v27267; GFX8-NEXT: v_mov_b32_e32 v4, v17268; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc7269; GFX8-NEXT: s_waitcnt vmcnt(0)7270; GFX8-NEXT: buffer_wbinvl17271; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v27272; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7273; GFX8-NEXT: v_mov_b32_e32 v2, v47274; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]7275; GFX8-NEXT: s_cbranch_execnz .LBB20_17276; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7277; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]7278; GFX8-NEXT: s_setpc_b64 s[30:31]7279;7280; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:7281; GFX7: ; %bb.0:7282; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7283; GFX7-NEXT: v_mov_b32_e32 v2, s207284; GFX7-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:10247285; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v17286; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v07287; GFX7-NEXT: s_mov_b64 s[4:5], 07288; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v17289; GFX7-NEXT: s_waitcnt vmcnt(0)7290; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v27291; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v27292; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v17293; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v57294; GFX7-NEXT: v_mov_b32_e32 v2, s207295; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start7296; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17297; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v47298; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v37299; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v47300; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v37301; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v47302; GFX7-NEXT: v_add_f32_e32 v5, v5, v07303; GFX7-NEXT: v_add_f32_e32 v6, v6, v17304; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v57305; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v67306; GFX7-NEXT: v_or_b32_e32 v5, v3, v47307; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v77308; GFX7-NEXT: v_or_b32_e32 v4, v6, v37309; GFX7-NEXT: v_mov_b32_e32 v7, v57310; GFX7-NEXT: v_mov_b32_e32 v6, v47311; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v2, s[16:19], 0 offen offset:1024 glc7312; GFX7-NEXT: s_waitcnt vmcnt(0)7313; GFX7-NEXT: buffer_wbinvl17314; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v67315; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v67316; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v47317; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v57318; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7319; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]7320; GFX7-NEXT: s_cbranch_execnz .LBB20_17321; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7322; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]7323; GFX7-NEXT: s_setpc_b64 s[30:31]7324;7325; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fine_grained_memory:7326; GFX6: ; %bb.0:7327; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7328; GFX6-NEXT: v_mov_b32_e32 v2, s207329; GFX6-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:10247330; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v17331; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v07332; GFX6-NEXT: s_add_i32 s6, s20, 0x4007333; GFX6-NEXT: s_mov_b64 s[4:5], 07334; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v17335; GFX6-NEXT: s_waitcnt vmcnt(0)7336; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v27337; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v27338; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v17339; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v57340; GFX6-NEXT: v_mov_b32_e32 v2, s67341; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start7342; GFX6-NEXT: ; =>This Inner Loop Header: Depth=17343; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v47344; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v37345; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v47346; GFX6-NEXT: s_waitcnt expcnt(0)7347; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v37348; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v47349; GFX6-NEXT: v_add_f32_e32 v5, v5, v07350; GFX6-NEXT: v_add_f32_e32 v6, v6, v17351; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v57352; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v67353; GFX6-NEXT: v_or_b32_e32 v5, v3, v47354; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v77355; GFX6-NEXT: v_or_b32_e32 v4, v6, v37356; GFX6-NEXT: v_mov_b32_e32 v7, v57357; GFX6-NEXT: v_mov_b32_e32 v6, v47358; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v2, s[16:19], 0 offen glc7359; GFX6-NEXT: s_waitcnt vmcnt(0)7360; GFX6-NEXT: buffer_wbinvl17361; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v67362; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v67363; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v47364; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v57365; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7366; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]7367; GFX6-NEXT: s_cbranch_execnz .LBB20_17368; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end7369; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]7370; GFX6-NEXT: s_waitcnt expcnt(0)7371; GFX6-NEXT: s_setpc_b64 s[30:31]7372 %gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 2567373 %unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !07374 ret void7375}7376 7377define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, <2 x half> %val) #0 {7378; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:7379; GFX12: ; %bb.0:7380; GFX12-NEXT: s_wait_loadcnt_dscnt 0x07381; GFX12-NEXT: s_wait_expcnt 0x07382; GFX12-NEXT: s_wait_samplecnt 0x07383; GFX12-NEXT: s_wait_bvhcnt 0x07384; GFX12-NEXT: s_wait_kmcnt 0x07385; GFX12-NEXT: s_mov_b32 s1, exec_lo7386; GFX12-NEXT: s_wait_storecnt 0x07387; GFX12-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=17388; GFX12-NEXT: v_readfirstlane_b32 s4, v07389; GFX12-NEXT: v_readfirstlane_b32 s5, v17390; GFX12-NEXT: v_readfirstlane_b32 s6, v27391; GFX12-NEXT: v_readfirstlane_b32 s7, v37392; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)7393; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)7394; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]7395; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]7396; GFX12-NEXT: s_and_b32 s0, vcc_lo, s07397; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)7398; GFX12-NEXT: s_and_saveexec_b32 s0, s07399; GFX12-NEXT: s_wait_loadcnt 0x07400; GFX12-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN7401; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr37402; GFX12-NEXT: ; implicit-def: $vgpr47403; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s07404; GFX12-NEXT: s_cbranch_execnz .LBB21_17405; GFX12-NEXT: ; %bb.2:7406; GFX12-NEXT: s_mov_b32 exec_lo, s17407; GFX12-NEXT: s_wait_loadcnt 0x07408; GFX12-NEXT: v_mov_b32_e32 v0, v57409; GFX12-NEXT: global_inv scope:SCOPE_DEV7410; GFX12-NEXT: s_wait_loadcnt 0x07411; GFX12-NEXT: s_setpc_b64 s[30:31]7412;7413; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:7414; GFX942: ; %bb.0:7415; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7416; GFX942-NEXT: s_mov_b64 s[2:3], exec7417; GFX942-NEXT: buffer_wbl2 sc17418; GFX942-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=17419; GFX942-NEXT: v_readfirstlane_b32 s4, v07420; GFX942-NEXT: v_readfirstlane_b32 s5, v17421; GFX942-NEXT: v_readfirstlane_b32 s6, v27422; GFX942-NEXT: v_readfirstlane_b32 s7, v37423; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]7424; GFX942-NEXT: s_nop 07425; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]7426; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]7427; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]7428; GFX942-NEXT: s_waitcnt vmcnt(0)7429; GFX942-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[4:7], 0 offen offset:1024 sc07430; GFX942-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr37431; GFX942-NEXT: ; implicit-def: $vgpr47432; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]7433; GFX942-NEXT: s_cbranch_execnz .LBB21_17434; GFX942-NEXT: ; %bb.2:7435; GFX942-NEXT: s_mov_b64 exec, s[2:3]7436; GFX942-NEXT: s_waitcnt vmcnt(0)7437; GFX942-NEXT: v_mov_b32_e32 v0, v57438; GFX942-NEXT: buffer_inv sc17439; GFX942-NEXT: s_setpc_b64 s[30:31]7440;7441; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:7442; GFX11: ; %bb.0:7443; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7444; GFX11-NEXT: s_mov_b32 s1, 07445; GFX11-NEXT: s_mov_b32 s2, exec_lo7446; GFX11-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=17447; GFX11-NEXT: v_readfirstlane_b32 s4, v07448; GFX11-NEXT: v_readfirstlane_b32 s5, v17449; GFX11-NEXT: v_readfirstlane_b32 s6, v27450; GFX11-NEXT: v_readfirstlane_b32 s7, v37451; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)7452; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]7453; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]7454; GFX11-NEXT: s_and_b32 s0, vcc_lo, s07455; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7456; GFX11-NEXT: s_and_saveexec_b32 s0, s07457; GFX11-NEXT: buffer_load_b32 v8, v4, s[4:7], 0 offen offset:10247458; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s07459; GFX11-NEXT: s_cbranch_execnz .LBB21_17460; GFX11-NEXT: ; %bb.2:7461; GFX11-NEXT: s_mov_b32 exec_lo, s27462; GFX11-NEXT: .p2align 67463; GFX11-NEXT: .LBB21_3: ; %atomicrmw.start7464; GFX11-NEXT: ; =>This Loop Header: Depth=17465; GFX11-NEXT: ; Child Loop BB21_4 Depth 27466; GFX11-NEXT: s_waitcnt vmcnt(0)7467; GFX11-NEXT: v_pk_add_f16 v7, v8, v57468; GFX11-NEXT: s_mov_b32 s2, exec_lo7469; GFX11-NEXT: s_waitcnt_vscnt null, 0x07470; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)7471; GFX11-NEXT: v_mov_b32_e32 v6, v77472; GFX11-NEXT: v_mov_b32_e32 v7, v87473; GFX11-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=17474; GFX11-NEXT: ; => This Inner Loop Header: Depth=27475; GFX11-NEXT: v_readfirstlane_b32 s4, v07476; GFX11-NEXT: v_readfirstlane_b32 s5, v17477; GFX11-NEXT: v_readfirstlane_b32 s6, v27478; GFX11-NEXT: v_readfirstlane_b32 s7, v37479; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)7480; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]7481; GFX11-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]7482; GFX11-NEXT: s_and_b32 s0, vcc_lo, s07483; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7484; GFX11-NEXT: s_and_saveexec_b32 s0, s07485; GFX11-NEXT: s_waitcnt vmcnt(0)7486; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[4:7], 0 offen offset:1024 glc7487; GFX11-NEXT: s_xor_b32 exec_lo, exec_lo, s07488; GFX11-NEXT: s_cbranch_execnz .LBB21_47489; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=17490; GFX11-NEXT: s_mov_b32 exec_lo, s27491; GFX11-NEXT: s_waitcnt vmcnt(0)7492; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v87493; GFX11-NEXT: v_mov_b32_e32 v8, v67494; GFX11-NEXT: buffer_gl1_inv7495; GFX11-NEXT: buffer_gl0_inv7496; GFX11-NEXT: s_or_b32 s1, vcc_lo, s17497; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7498; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s17499; GFX11-NEXT: s_cbranch_execnz .LBB21_37500; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end7501; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s17502; GFX11-NEXT: v_mov_b32_e32 v0, v67503; GFX11-NEXT: s_setpc_b64 s[30:31]7504;7505; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:7506; GFX10: ; %bb.0:7507; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7508; GFX10-NEXT: s_mov_b32 s5, 07509; GFX10-NEXT: s_mov_b32 s6, exec_lo7510; GFX10-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=17511; GFX10-NEXT: v_readfirstlane_b32 s8, v07512; GFX10-NEXT: v_readfirstlane_b32 s9, v17513; GFX10-NEXT: v_readfirstlane_b32 s10, v27514; GFX10-NEXT: v_readfirstlane_b32 s11, v37515; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]7516; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]7517; GFX10-NEXT: s_and_b32 s4, vcc_lo, s47518; GFX10-NEXT: s_and_saveexec_b32 s4, s47519; GFX10-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:10247520; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)7521; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s47522; GFX10-NEXT: s_cbranch_execnz .LBB21_17523; GFX10-NEXT: ; %bb.2:7524; GFX10-NEXT: s_mov_b32 exec_lo, s67525; GFX10-NEXT: .LBB21_3: ; %atomicrmw.start7526; GFX10-NEXT: ; =>This Loop Header: Depth=17527; GFX10-NEXT: ; Child Loop BB21_4 Depth 27528; GFX10-NEXT: s_waitcnt vmcnt(0)7529; GFX10-NEXT: v_pk_add_f16 v7, v8, v57530; GFX10-NEXT: s_mov_b32 s6, exec_lo7531; GFX10-NEXT: s_waitcnt_vscnt null, 0x07532; GFX10-NEXT: v_mov_b32_e32 v6, v77533; GFX10-NEXT: v_mov_b32_e32 v7, v87534; GFX10-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=17535; GFX10-NEXT: ; => This Inner Loop Header: Depth=27536; GFX10-NEXT: v_readfirstlane_b32 s8, v07537; GFX10-NEXT: v_readfirstlane_b32 s9, v17538; GFX10-NEXT: v_readfirstlane_b32 s10, v27539; GFX10-NEXT: v_readfirstlane_b32 s11, v37540; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]7541; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]7542; GFX10-NEXT: s_and_b32 s4, vcc_lo, s47543; GFX10-NEXT: s_and_saveexec_b32 s4, s47544; GFX10-NEXT: s_waitcnt vmcnt(0)7545; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc7546; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)7547; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s47548; GFX10-NEXT: s_cbranch_execnz .LBB21_47549; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=17550; GFX10-NEXT: s_mov_b32 exec_lo, s67551; GFX10-NEXT: s_waitcnt vmcnt(0)7552; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v87553; GFX10-NEXT: v_mov_b32_e32 v8, v67554; GFX10-NEXT: buffer_gl1_inv7555; GFX10-NEXT: buffer_gl0_inv7556; GFX10-NEXT: s_or_b32 s5, vcc_lo, s57557; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)7558; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s57559; GFX10-NEXT: s_cbranch_execnz .LBB21_37560; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end7561; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s57562; GFX10-NEXT: v_mov_b32_e32 v0, v67563; GFX10-NEXT: s_setpc_b64 s[30:31]7564;7565; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:7566; GFX90A: ; %bb.0:7567; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7568; GFX90A-NEXT: s_mov_b64 s[6:7], exec7569; GFX90A-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=17570; GFX90A-NEXT: v_readfirstlane_b32 s8, v07571; GFX90A-NEXT: v_readfirstlane_b32 s9, v17572; GFX90A-NEXT: v_readfirstlane_b32 s10, v27573; GFX90A-NEXT: v_readfirstlane_b32 s11, v37574; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]7575; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]7576; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]7577; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]7578; GFX90A-NEXT: s_waitcnt vmcnt(0)7579; GFX90A-NEXT: buffer_atomic_pk_add_f16 v5, v4, s[8:11], 0 offen offset:1024 glc7580; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr37581; GFX90A-NEXT: ; implicit-def: $vgpr47582; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]7583; GFX90A-NEXT: s_cbranch_execnz .LBB21_17584; GFX90A-NEXT: ; %bb.2:7585; GFX90A-NEXT: s_mov_b64 exec, s[6:7]7586; GFX90A-NEXT: s_waitcnt vmcnt(0)7587; GFX90A-NEXT: v_mov_b32_e32 v0, v57588; GFX90A-NEXT: buffer_wbinvl17589; GFX90A-NEXT: s_setpc_b64 s[30:31]7590;7591; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:7592; GFX908: ; %bb.0:7593; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7594; GFX908-NEXT: s_mov_b64 s[6:7], exec7595; GFX908-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=17596; GFX908-NEXT: v_readfirstlane_b32 s8, v07597; GFX908-NEXT: v_readfirstlane_b32 s9, v17598; GFX908-NEXT: v_readfirstlane_b32 s10, v27599; GFX908-NEXT: v_readfirstlane_b32 s11, v37600; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]7601; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]7602; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]7603; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]7604; GFX908-NEXT: s_nop 07605; GFX908-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:10247606; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]7607; GFX908-NEXT: s_cbranch_execnz .LBB21_17608; GFX908-NEXT: ; %bb.2:7609; GFX908-NEXT: s_mov_b64 exec, s[6:7]7610; GFX908-NEXT: s_mov_b64 s[6:7], 07611; GFX908-NEXT: .LBB21_3: ; %atomicrmw.start7612; GFX908-NEXT: ; =>This Loop Header: Depth=17613; GFX908-NEXT: ; Child Loop BB21_4 Depth 27614; GFX908-NEXT: s_waitcnt vmcnt(0)7615; GFX908-NEXT: v_pk_add_f16 v7, v8, v57616; GFX908-NEXT: v_mov_b32_e32 v6, v77617; GFX908-NEXT: s_mov_b64 s[12:13], exec7618; GFX908-NEXT: v_mov_b32_e32 v7, v87619; GFX908-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=17620; GFX908-NEXT: ; => This Inner Loop Header: Depth=27621; GFX908-NEXT: v_readfirstlane_b32 s8, v07622; GFX908-NEXT: v_readfirstlane_b32 s9, v17623; GFX908-NEXT: v_readfirstlane_b32 s10, v27624; GFX908-NEXT: v_readfirstlane_b32 s11, v37625; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]7626; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]7627; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]7628; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]7629; GFX908-NEXT: s_waitcnt vmcnt(0)7630; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc7631; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]7632; GFX908-NEXT: s_cbranch_execnz .LBB21_47633; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=17634; GFX908-NEXT: s_mov_b64 exec, s[12:13]7635; GFX908-NEXT: s_waitcnt vmcnt(0)7636; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v87637; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7638; GFX908-NEXT: v_mov_b32_e32 v8, v67639; GFX908-NEXT: buffer_wbinvl17640; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]7641; GFX908-NEXT: s_cbranch_execnz .LBB21_37642; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end7643; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]7644; GFX908-NEXT: v_mov_b32_e32 v0, v67645; GFX908-NEXT: s_setpc_b64 s[30:31]7646;7647; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:7648; GFX8: ; %bb.0:7649; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7650; GFX8-NEXT: s_mov_b64 s[6:7], exec7651; GFX8-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=17652; GFX8-NEXT: v_readfirstlane_b32 s8, v07653; GFX8-NEXT: v_readfirstlane_b32 s9, v17654; GFX8-NEXT: v_readfirstlane_b32 s10, v27655; GFX8-NEXT: v_readfirstlane_b32 s11, v37656; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]7657; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]7658; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]7659; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]7660; GFX8-NEXT: s_nop 07661; GFX8-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:10247662; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]7663; GFX8-NEXT: s_cbranch_execnz .LBB21_17664; GFX8-NEXT: ; %bb.2:7665; GFX8-NEXT: s_mov_b64 exec, s[6:7]7666; GFX8-NEXT: s_mov_b64 s[6:7], 07667; GFX8-NEXT: .LBB21_3: ; %atomicrmw.start7668; GFX8-NEXT: ; =>This Loop Header: Depth=17669; GFX8-NEXT: ; Child Loop BB21_4 Depth 27670; GFX8-NEXT: s_waitcnt vmcnt(0)7671; GFX8-NEXT: v_add_f16_sdwa v6, v8, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_17672; GFX8-NEXT: v_add_f16_e32 v7, v8, v57673; GFX8-NEXT: v_or_b32_e32 v7, v7, v67674; GFX8-NEXT: v_mov_b32_e32 v6, v77675; GFX8-NEXT: s_mov_b64 s[12:13], exec7676; GFX8-NEXT: v_mov_b32_e32 v7, v87677; GFX8-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=17678; GFX8-NEXT: ; => This Inner Loop Header: Depth=27679; GFX8-NEXT: v_readfirstlane_b32 s8, v07680; GFX8-NEXT: v_readfirstlane_b32 s9, v17681; GFX8-NEXT: v_readfirstlane_b32 s10, v27682; GFX8-NEXT: v_readfirstlane_b32 s11, v37683; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]7684; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]7685; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]7686; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]7687; GFX8-NEXT: s_waitcnt vmcnt(0)7688; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc7689; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]7690; GFX8-NEXT: s_cbranch_execnz .LBB21_47691; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=17692; GFX8-NEXT: s_mov_b64 exec, s[12:13]7693; GFX8-NEXT: s_waitcnt vmcnt(0)7694; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v87695; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7696; GFX8-NEXT: v_mov_b32_e32 v8, v67697; GFX8-NEXT: buffer_wbinvl17698; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]7699; GFX8-NEXT: s_cbranch_execnz .LBB21_37700; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end7701; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]7702; GFX8-NEXT: v_mov_b32_e32 v0, v67703; GFX8-NEXT: s_setpc_b64 s[30:31]7704;7705; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:7706; GFX7: ; %bb.0:7707; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7708; GFX7-NEXT: s_mov_b64 s[6:7], exec7709; GFX7-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=17710; GFX7-NEXT: v_readfirstlane_b32 s8, v07711; GFX7-NEXT: v_readfirstlane_b32 s9, v17712; GFX7-NEXT: v_readfirstlane_b32 s10, v27713; GFX7-NEXT: v_readfirstlane_b32 s11, v37714; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]7715; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]7716; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]7717; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]7718; GFX7-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:10247719; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]7720; GFX7-NEXT: s_cbranch_execnz .LBB21_17721; GFX7-NEXT: ; %bb.2:7722; GFX7-NEXT: s_mov_b64 exec, s[6:7]7723; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v67724; GFX7-NEXT: v_cvt_f16_f32_e32 v9, v57725; GFX7-NEXT: s_waitcnt vmcnt(0)7726; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v87727; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v87728; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v57729; GFX7-NEXT: v_cvt_f32_f16_e32 v10, v67730; GFX7-NEXT: v_cvt_f32_f16_e32 v11, v97731; GFX7-NEXT: s_mov_b64 s[6:7], 07732; GFX7-NEXT: .LBB21_3: ; %atomicrmw.start7733; GFX7-NEXT: ; =>This Loop Header: Depth=17734; GFX7-NEXT: ; Child Loop BB21_4 Depth 27735; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v57736; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v77737; GFX7-NEXT: s_mov_b64 s[12:13], exec7738; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v57739; GFX7-NEXT: v_cvt_f32_f16_e32 v8, v67740; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v57741; GFX7-NEXT: v_or_b32_e32 v6, v6, v57742; GFX7-NEXT: v_add_f32_e32 v7, v7, v107743; GFX7-NEXT: v_add_f32_e32 v8, v8, v117744; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v77745; GFX7-NEXT: v_cvt_f16_f32_e32 v8, v87746; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v77747; GFX7-NEXT: v_or_b32_e32 v5, v8, v57748; GFX7-NEXT: v_mov_b32_e32 v9, v67749; GFX7-NEXT: v_mov_b32_e32 v8, v57750; GFX7-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=17751; GFX7-NEXT: ; => This Inner Loop Header: Depth=27752; GFX7-NEXT: v_readfirstlane_b32 s8, v07753; GFX7-NEXT: v_readfirstlane_b32 s9, v17754; GFX7-NEXT: v_readfirstlane_b32 s10, v27755; GFX7-NEXT: v_readfirstlane_b32 s11, v37756; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]7757; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]7758; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]7759; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]7760; GFX7-NEXT: s_waitcnt vmcnt(0)7761; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v4, s[8:11], 0 offen offset:1024 glc7762; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]7763; GFX7-NEXT: s_cbranch_execnz .LBB21_47764; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=17765; GFX7-NEXT: s_mov_b64 exec, s[12:13]7766; GFX7-NEXT: s_waitcnt vmcnt(0)7767; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v87768; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v87769; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v57770; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v67771; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7772; GFX7-NEXT: buffer_wbinvl17773; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]7774; GFX7-NEXT: s_cbranch_execnz .LBB21_37775; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end7776; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]7777; GFX7-NEXT: v_mov_b32_e32 v0, v77778; GFX7-NEXT: v_mov_b32_e32 v1, v57779; GFX7-NEXT: s_setpc_b64 s[30:31]7780;7781; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:7782; GFX6: ; %bb.0:7783; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7784; GFX6-NEXT: v_add_i32_e32 v9, vcc, 0x400, v47785; GFX6-NEXT: s_mov_b64 s[6:7], exec7786; GFX6-NEXT: .LBB21_1: ; =>This Inner Loop Header: Depth=17787; GFX6-NEXT: v_readfirstlane_b32 s8, v07788; GFX6-NEXT: v_readfirstlane_b32 s9, v17789; GFX6-NEXT: v_readfirstlane_b32 s10, v27790; GFX6-NEXT: v_readfirstlane_b32 s11, v37791; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]7792; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]7793; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]7794; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]7795; GFX6-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:10247796; GFX6-NEXT: ; implicit-def: $vgpr47797; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]7798; GFX6-NEXT: s_cbranch_execnz .LBB21_17799; GFX6-NEXT: ; %bb.2:7800; GFX6-NEXT: s_mov_b64 exec, s[6:7]7801; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v67802; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v57803; GFX6-NEXT: s_waitcnt vmcnt(0)7804; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v77805; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v77806; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v57807; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v67808; GFX6-NEXT: v_cvt_f32_f16_e32 v11, v87809; GFX6-NEXT: s_mov_b64 s[6:7], 07810; GFX6-NEXT: .LBB21_3: ; %atomicrmw.start7811; GFX6-NEXT: ; =>This Loop Header: Depth=17812; GFX6-NEXT: ; Child Loop BB21_4 Depth 27813; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v57814; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v47815; GFX6-NEXT: s_mov_b64 s[12:13], exec7816; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v57817; GFX6-NEXT: s_waitcnt expcnt(0)7818; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v47819; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v57820; GFX6-NEXT: v_add_f32_e32 v6, v6, v107821; GFX6-NEXT: v_add_f32_e32 v7, v7, v117822; GFX6-NEXT: v_cvt_f16_f32_e32 v8, v67823; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v77824; GFX6-NEXT: v_or_b32_e32 v6, v4, v57825; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v87826; GFX6-NEXT: v_or_b32_e32 v5, v7, v47827; GFX6-NEXT: v_mov_b32_e32 v8, v67828; GFX6-NEXT: v_mov_b32_e32 v7, v57829; GFX6-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=17830; GFX6-NEXT: ; => This Inner Loop Header: Depth=27831; GFX6-NEXT: v_readfirstlane_b32 s8, v07832; GFX6-NEXT: v_readfirstlane_b32 s9, v17833; GFX6-NEXT: v_readfirstlane_b32 s10, v27834; GFX6-NEXT: v_readfirstlane_b32 s11, v37835; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]7836; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]7837; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]7838; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]7839; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)7840; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v9, s[8:11], 0 offen glc7841; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]7842; GFX6-NEXT: s_cbranch_execnz .LBB21_47843; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB21_3 Depth=17844; GFX6-NEXT: s_mov_b64 exec, s[12:13]7845; GFX6-NEXT: s_waitcnt vmcnt(0)7846; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v77847; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v77848; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v57849; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v67850; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7851; GFX6-NEXT: buffer_wbinvl17852; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]7853; GFX6-NEXT: s_cbranch_execnz .LBB21_37854; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end7855; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]7856; GFX6-NEXT: v_mov_b32_e32 v0, v47857; GFX6-NEXT: v_mov_b32_e32 v1, v57858; GFX6-NEXT: s_waitcnt expcnt(0)7859; GFX6-NEXT: s_setpc_b64 s[30:31]7860 %gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 2567861 %result = atomicrmw fadd ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !07862 ret <2 x half> %result7863}7864 7865define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrspace(7) inreg %ptr, <2 x half> %val) {7866; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:7867; GFX12: ; %bb.0:7868; GFX12-NEXT: s_wait_loadcnt_dscnt 0x07869; GFX12-NEXT: s_wait_expcnt 0x07870; GFX12-NEXT: s_wait_samplecnt 0x07871; GFX12-NEXT: s_wait_bvhcnt 0x07872; GFX12-NEXT: s_wait_kmcnt 0x07873; GFX12-NEXT: v_mov_b32_e32 v1, s167874; GFX12-NEXT: s_wait_storecnt 0x07875; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN7876; GFX12-NEXT: s_wait_loadcnt 0x07877; GFX12-NEXT: global_inv scope:SCOPE_DEV7878; GFX12-NEXT: s_wait_loadcnt 0x07879; GFX12-NEXT: s_setpc_b64 s[30:31]7880;7881; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:7882; GFX942: ; %bb.0:7883; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7884; GFX942-NEXT: v_mov_b32_e32 v1, s167885; GFX942-NEXT: buffer_wbl2 sc17886; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:1024 sc07887; GFX942-NEXT: s_waitcnt vmcnt(0)7888; GFX942-NEXT: buffer_inv sc17889; GFX942-NEXT: s_setpc_b64 s[30:31]7890;7891; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:7892; GFX11: ; %bb.0:7893; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7894; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s167895; GFX11-NEXT: v_mov_b32_e32 v0, s167896; GFX11-NEXT: s_mov_b32 s4, 07897; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:10247898; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start7899; GFX11-NEXT: ; =>This Inner Loop Header: Depth=17900; GFX11-NEXT: s_waitcnt vmcnt(0)7901; GFX11-NEXT: v_mov_b32_e32 v5, v07902; GFX11-NEXT: s_waitcnt_vscnt null, 0x07903; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7904; GFX11-NEXT: v_pk_add_f16 v4, v5, v27905; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v57906; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc7907; GFX11-NEXT: s_waitcnt vmcnt(0)7908; GFX11-NEXT: buffer_gl1_inv7909; GFX11-NEXT: buffer_gl0_inv7910; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v57911; GFX11-NEXT: s_or_b32 s4, vcc_lo, s47912; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7913; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s47914; GFX11-NEXT: s_cbranch_execnz .LBB22_17915; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end7916; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s47917; GFX11-NEXT: s_setpc_b64 s[30:31]7918;7919; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:7920; GFX10: ; %bb.0:7921; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7922; GFX10-NEXT: v_mov_b32_e32 v2, v07923; GFX10-NEXT: v_mov_b32_e32 v0, s207924; GFX10-NEXT: v_mov_b32_e32 v3, s207925; GFX10-NEXT: s_mov_b32 s4, 07926; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10247927; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start7928; GFX10-NEXT: ; =>This Inner Loop Header: Depth=17929; GFX10-NEXT: s_waitcnt vmcnt(0)7930; GFX10-NEXT: v_mov_b32_e32 v5, v07931; GFX10-NEXT: s_waitcnt_vscnt null, 0x07932; GFX10-NEXT: v_pk_add_f16 v4, v5, v27933; GFX10-NEXT: v_mov_b32_e32 v0, v47934; GFX10-NEXT: v_mov_b32_e32 v1, v57935; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc7936; GFX10-NEXT: s_waitcnt vmcnt(0)7937; GFX10-NEXT: buffer_gl1_inv7938; GFX10-NEXT: buffer_gl0_inv7939; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v57940; GFX10-NEXT: s_or_b32 s4, vcc_lo, s47941; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s47942; GFX10-NEXT: s_cbranch_execnz .LBB22_17943; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end7944; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s47945; GFX10-NEXT: s_setpc_b64 s[30:31]7946;7947; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:7948; GFX90A: ; %bb.0:7949; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7950; GFX90A-NEXT: v_mov_b32_e32 v2, v07951; GFX90A-NEXT: v_mov_b32_e32 v0, s207952; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10247953; GFX90A-NEXT: s_mov_b64 s[4:5], 07954; GFX90A-NEXT: v_mov_b32_e32 v3, s207955; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start7956; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=17957; GFX90A-NEXT: s_waitcnt vmcnt(0)7958; GFX90A-NEXT: v_mov_b32_e32 v5, v07959; GFX90A-NEXT: v_pk_add_f16 v4, v5, v27960; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]7961; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc7962; GFX90A-NEXT: s_waitcnt vmcnt(0)7963; GFX90A-NEXT: buffer_wbinvl17964; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v57965; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7966; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]7967; GFX90A-NEXT: s_cbranch_execnz .LBB22_17968; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end7969; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]7970; GFX90A-NEXT: s_setpc_b64 s[30:31]7971;7972; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:7973; GFX908: ; %bb.0:7974; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7975; GFX908-NEXT: v_mov_b32_e32 v2, v07976; GFX908-NEXT: v_mov_b32_e32 v0, s207977; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10247978; GFX908-NEXT: s_mov_b64 s[4:5], 07979; GFX908-NEXT: v_mov_b32_e32 v3, s207980; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start7981; GFX908-NEXT: ; =>This Inner Loop Header: Depth=17982; GFX908-NEXT: s_waitcnt vmcnt(0)7983; GFX908-NEXT: v_mov_b32_e32 v5, v07984; GFX908-NEXT: v_pk_add_f16 v4, v5, v27985; GFX908-NEXT: v_mov_b32_e32 v0, v47986; GFX908-NEXT: v_mov_b32_e32 v1, v57987; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc7988; GFX908-NEXT: s_waitcnt vmcnt(0)7989; GFX908-NEXT: buffer_wbinvl17990; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v57991; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7992; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]7993; GFX908-NEXT: s_cbranch_execnz .LBB22_17994; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end7995; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]7996; GFX908-NEXT: s_setpc_b64 s[30:31]7997;7998; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:7999; GFX8: ; %bb.0:8000; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8001; GFX8-NEXT: v_mov_b32_e32 v2, v08002; GFX8-NEXT: v_mov_b32_e32 v0, s208003; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10248004; GFX8-NEXT: s_mov_b64 s[4:5], 08005; GFX8-NEXT: v_mov_b32_e32 v3, s208006; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start8007; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18008; GFX8-NEXT: s_waitcnt vmcnt(0)8009; GFX8-NEXT: v_mov_b32_e32 v5, v08010; GFX8-NEXT: v_add_f16_sdwa v0, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_18011; GFX8-NEXT: v_add_f16_e32 v1, v5, v28012; GFX8-NEXT: v_or_b32_e32 v4, v1, v08013; GFX8-NEXT: v_mov_b32_e32 v0, v48014; GFX8-NEXT: v_mov_b32_e32 v1, v58015; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc8016; GFX8-NEXT: s_waitcnt vmcnt(0)8017; GFX8-NEXT: buffer_wbinvl18018; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v58019; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8020; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]8021; GFX8-NEXT: s_cbranch_execnz .LBB22_18022; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8023; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]8024; GFX8-NEXT: s_setpc_b64 s[30:31]8025;8026; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:8027; GFX7: ; %bb.0:8028; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8029; GFX7-NEXT: v_mov_b32_e32 v2, s208030; GFX7-NEXT: buffer_load_dword v3, v2, s[16:19], 0 offen offset:10248031; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v18032; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v08033; GFX7-NEXT: s_mov_b64 s[4:5], 08034; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v18035; GFX7-NEXT: s_waitcnt vmcnt(0)8036; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v38037; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v38038; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v18039; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v48040; GFX7-NEXT: v_mov_b32_e32 v4, s208041; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start8042; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18043; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v18044; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v08045; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v18046; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v08047; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v18048; GFX7-NEXT: v_add_f32_e32 v5, v5, v28049; GFX7-NEXT: v_add_f32_e32 v6, v6, v38050; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v58051; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v68052; GFX7-NEXT: v_or_b32_e32 v6, v0, v18053; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v58054; GFX7-NEXT: v_or_b32_e32 v5, v7, v08055; GFX7-NEXT: v_mov_b32_e32 v8, v68056; GFX7-NEXT: v_mov_b32_e32 v7, v58057; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[16:19], 0 offen offset:1024 glc8058; GFX7-NEXT: s_waitcnt vmcnt(0)8059; GFX7-NEXT: buffer_wbinvl18060; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v78061; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v78062; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v18063; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v68064; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8065; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]8066; GFX7-NEXT: s_cbranch_execnz .LBB22_18067; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8068; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]8069; GFX7-NEXT: s_setpc_b64 s[30:31]8070;8071; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:8072; GFX6: ; %bb.0:8073; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8074; GFX6-NEXT: v_mov_b32_e32 v2, s208075; GFX6-NEXT: buffer_load_dword v3, v2, s[16:19], 0 offen offset:10248076; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v18077; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v08078; GFX6-NEXT: s_add_i32 s6, s20, 0x4008079; GFX6-NEXT: s_mov_b64 s[4:5], 08080; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v18081; GFX6-NEXT: s_waitcnt vmcnt(0)8082; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v38083; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v38084; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v18085; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v48086; GFX6-NEXT: v_mov_b32_e32 v4, s68087; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start8088; GFX6-NEXT: ; =>This Inner Loop Header: Depth=18089; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v18090; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v08091; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v18092; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v08093; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v18094; GFX6-NEXT: v_add_f32_e32 v5, v5, v28095; GFX6-NEXT: v_add_f32_e32 v6, v6, v38096; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v58097; GFX6-NEXT: s_waitcnt expcnt(0)8098; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v68099; GFX6-NEXT: v_or_b32_e32 v6, v0, v18100; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v58101; GFX6-NEXT: v_or_b32_e32 v5, v7, v08102; GFX6-NEXT: v_mov_b32_e32 v8, v68103; GFX6-NEXT: v_mov_b32_e32 v7, v58104; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[16:19], 0 offen glc8105; GFX6-NEXT: s_waitcnt vmcnt(0)8106; GFX6-NEXT: buffer_wbinvl18107; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v78108; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v78109; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v18110; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v68111; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8112; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]8113; GFX6-NEXT: s_cbranch_execnz .LBB22_18114; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end8115; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]8116; GFX6-NEXT: s_waitcnt expcnt(0)8117; GFX6-NEXT: s_setpc_b64 s[30:31]8118 %gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 2568119 %result = atomicrmw fadd ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst8120 ret <2 x half> %result8121}8122 8123define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(7) inreg %ptr, <2 x half> %val) {8124; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:8125; GFX12: ; %bb.0:8126; GFX12-NEXT: s_wait_loadcnt_dscnt 0x08127; GFX12-NEXT: s_wait_expcnt 0x08128; GFX12-NEXT: s_wait_samplecnt 0x08129; GFX12-NEXT: s_wait_bvhcnt 0x08130; GFX12-NEXT: s_wait_kmcnt 0x08131; GFX12-NEXT: v_mov_b32_e32 v1, s168132; GFX12-NEXT: s_wait_storecnt 0x08133; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:10248134; GFX12-NEXT: s_wait_storecnt 0x08135; GFX12-NEXT: global_inv scope:SCOPE_DEV8136; GFX12-NEXT: s_wait_loadcnt 0x08137; GFX12-NEXT: s_setpc_b64 s[30:31]8138;8139; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:8140; GFX942: ; %bb.0:8141; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8142; GFX942-NEXT: v_mov_b32_e32 v1, s168143; GFX942-NEXT: buffer_wbl2 sc18144; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:10248145; GFX942-NEXT: s_waitcnt vmcnt(0)8146; GFX942-NEXT: buffer_inv sc18147; GFX942-NEXT: s_setpc_b64 s[30:31]8148;8149; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:8150; GFX11: ; %bb.0:8151; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8152; GFX11-NEXT: v_mov_b32_e32 v1, s168153; GFX11-NEXT: v_mov_b32_e32 v3, s168154; GFX11-NEXT: s_mov_b32 s4, 08155; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen offset:10248156; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start8157; GFX11-NEXT: ; =>This Inner Loop Header: Depth=18158; GFX11-NEXT: s_waitcnt vmcnt(0)8159; GFX11-NEXT: v_pk_add_f16 v1, v2, v08160; GFX11-NEXT: v_mov_b32_e32 v5, v28161; GFX11-NEXT: s_waitcnt_vscnt null, 0x08162; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)8163; GFX11-NEXT: v_mov_b32_e32 v4, v18164; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc8165; GFX11-NEXT: s_waitcnt vmcnt(0)8166; GFX11-NEXT: buffer_gl1_inv8167; GFX11-NEXT: buffer_gl0_inv8168; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v28169; GFX11-NEXT: v_mov_b32_e32 v2, v48170; GFX11-NEXT: s_or_b32 s4, vcc_lo, s48171; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8172; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s48173; GFX11-NEXT: s_cbranch_execnz .LBB23_18174; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end8175; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s48176; GFX11-NEXT: s_setpc_b64 s[30:31]8177;8178; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:8179; GFX10: ; %bb.0:8180; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8181; GFX10-NEXT: v_mov_b32_e32 v1, s208182; GFX10-NEXT: v_mov_b32_e32 v3, s208183; GFX10-NEXT: s_mov_b32 s4, 08184; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10248185; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start8186; GFX10-NEXT: ; =>This Inner Loop Header: Depth=18187; GFX10-NEXT: s_waitcnt vmcnt(0)8188; GFX10-NEXT: v_pk_add_f16 v1, v2, v08189; GFX10-NEXT: v_mov_b32_e32 v5, v28190; GFX10-NEXT: s_waitcnt_vscnt null, 0x08191; GFX10-NEXT: v_mov_b32_e32 v4, v18192; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc8193; GFX10-NEXT: s_waitcnt vmcnt(0)8194; GFX10-NEXT: buffer_gl1_inv8195; GFX10-NEXT: buffer_gl0_inv8196; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v28197; GFX10-NEXT: v_mov_b32_e32 v2, v48198; GFX10-NEXT: s_or_b32 s4, vcc_lo, s48199; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s48200; GFX10-NEXT: s_cbranch_execnz .LBB23_18201; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end8202; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s48203; GFX10-NEXT: s_setpc_b64 s[30:31]8204;8205; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:8206; GFX90A: ; %bb.0:8207; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8208; GFX90A-NEXT: v_mov_b32_e32 v1, s208209; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen offset:10248210; GFX90A-NEXT: s_mov_b64 s[4:5], 08211; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start8212; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=18213; GFX90A-NEXT: s_waitcnt vmcnt(0)8214; GFX90A-NEXT: v_pk_add_f16 v2, v3, v08215; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]8216; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc8217; GFX90A-NEXT: s_waitcnt vmcnt(0)8218; GFX90A-NEXT: buffer_wbinvl18219; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v38220; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8221; GFX90A-NEXT: v_mov_b32_e32 v3, v48222; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]8223; GFX90A-NEXT: s_cbranch_execnz .LBB23_18224; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end8225; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]8226; GFX90A-NEXT: s_setpc_b64 s[30:31]8227;8228; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:8229; GFX908: ; %bb.0:8230; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8231; GFX908-NEXT: v_mov_b32_e32 v1, s208232; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10248233; GFX908-NEXT: s_mov_b64 s[4:5], 08234; GFX908-NEXT: v_mov_b32_e32 v3, s208235; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start8236; GFX908-NEXT: ; =>This Inner Loop Header: Depth=18237; GFX908-NEXT: s_waitcnt vmcnt(0)8238; GFX908-NEXT: v_pk_add_f16 v1, v2, v08239; GFX908-NEXT: v_mov_b32_e32 v5, v28240; GFX908-NEXT: v_mov_b32_e32 v4, v18241; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc8242; GFX908-NEXT: s_waitcnt vmcnt(0)8243; GFX908-NEXT: buffer_wbinvl18244; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v28245; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8246; GFX908-NEXT: v_mov_b32_e32 v2, v48247; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]8248; GFX908-NEXT: s_cbranch_execnz .LBB23_18249; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end8250; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]8251; GFX908-NEXT: s_setpc_b64 s[30:31]8252;8253; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:8254; GFX8: ; %bb.0:8255; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8256; GFX8-NEXT: v_mov_b32_e32 v1, s208257; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10248258; GFX8-NEXT: s_mov_b64 s[4:5], 08259; GFX8-NEXT: v_mov_b32_e32 v3, s208260; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start8261; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18262; GFX8-NEXT: s_waitcnt vmcnt(0)8263; GFX8-NEXT: v_add_f16_sdwa v1, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_18264; GFX8-NEXT: v_add_f16_e32 v4, v2, v08265; GFX8-NEXT: v_or_b32_e32 v1, v4, v18266; GFX8-NEXT: v_mov_b32_e32 v5, v28267; GFX8-NEXT: v_mov_b32_e32 v4, v18268; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc8269; GFX8-NEXT: s_waitcnt vmcnt(0)8270; GFX8-NEXT: buffer_wbinvl18271; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v28272; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8273; GFX8-NEXT: v_mov_b32_e32 v2, v48274; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]8275; GFX8-NEXT: s_cbranch_execnz .LBB23_18276; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8277; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]8278; GFX8-NEXT: s_setpc_b64 s[30:31]8279;8280; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:8281; GFX7: ; %bb.0:8282; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8283; GFX7-NEXT: v_mov_b32_e32 v2, s208284; GFX7-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:10248285; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v18286; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v08287; GFX7-NEXT: s_mov_b64 s[4:5], 08288; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v18289; GFX7-NEXT: s_waitcnt vmcnt(0)8290; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v28291; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v28292; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v18293; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v58294; GFX7-NEXT: v_mov_b32_e32 v2, s208295; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start8296; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18297; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v48298; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v38299; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v48300; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v38301; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v48302; GFX7-NEXT: v_add_f32_e32 v5, v5, v08303; GFX7-NEXT: v_add_f32_e32 v6, v6, v18304; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v58305; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v68306; GFX7-NEXT: v_or_b32_e32 v5, v3, v48307; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v78308; GFX7-NEXT: v_or_b32_e32 v4, v6, v38309; GFX7-NEXT: v_mov_b32_e32 v7, v58310; GFX7-NEXT: v_mov_b32_e32 v6, v48311; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v2, s[16:19], 0 offen offset:1024 glc8312; GFX7-NEXT: s_waitcnt vmcnt(0)8313; GFX7-NEXT: buffer_wbinvl18314; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v68315; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v68316; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v48317; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v58318; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8319; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]8320; GFX7-NEXT: s_cbranch_execnz .LBB23_18321; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8322; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]8323; GFX7-NEXT: s_setpc_b64 s[30:31]8324;8325; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset:8326; GFX6: ; %bb.0:8327; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8328; GFX6-NEXT: v_mov_b32_e32 v2, s208329; GFX6-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:10248330; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v18331; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v08332; GFX6-NEXT: s_add_i32 s6, s20, 0x4008333; GFX6-NEXT: s_mov_b64 s[4:5], 08334; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v18335; GFX6-NEXT: s_waitcnt vmcnt(0)8336; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v28337; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v28338; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v18339; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v58340; GFX6-NEXT: v_mov_b32_e32 v2, s68341; GFX6-NEXT: .LBB23_1: ; %atomicrmw.start8342; GFX6-NEXT: ; =>This Inner Loop Header: Depth=18343; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v48344; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v38345; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v48346; GFX6-NEXT: s_waitcnt expcnt(0)8347; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v38348; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v48349; GFX6-NEXT: v_add_f32_e32 v5, v5, v08350; GFX6-NEXT: v_add_f32_e32 v6, v6, v18351; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v58352; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v68353; GFX6-NEXT: v_or_b32_e32 v5, v3, v48354; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v78355; GFX6-NEXT: v_or_b32_e32 v4, v6, v38356; GFX6-NEXT: v_mov_b32_e32 v7, v58357; GFX6-NEXT: v_mov_b32_e32 v6, v48358; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v2, s[16:19], 0 offen glc8359; GFX6-NEXT: s_waitcnt vmcnt(0)8360; GFX6-NEXT: buffer_wbinvl18361; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v68362; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v68363; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v48364; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v58365; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8366; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]8367; GFX6-NEXT: s_cbranch_execnz .LBB23_18368; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end8369; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]8370; GFX6-NEXT: s_waitcnt expcnt(0)8371; GFX6-NEXT: s_setpc_b64 s[30:31]8372 %gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 2568373 %unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst8374 ret void8375}8376 8377define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {8378; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:8379; GFX12: ; %bb.0:8380; GFX12-NEXT: s_wait_loadcnt_dscnt 0x08381; GFX12-NEXT: s_wait_expcnt 0x08382; GFX12-NEXT: s_wait_samplecnt 0x08383; GFX12-NEXT: s_wait_bvhcnt 0x08384; GFX12-NEXT: s_wait_kmcnt 0x08385; GFX12-NEXT: v_mov_b32_e32 v1, s168386; GFX12-NEXT: s_wait_storecnt 0x08387; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN8388; GFX12-NEXT: s_wait_loadcnt 0x08389; GFX12-NEXT: global_inv scope:SCOPE_DEV8390; GFX12-NEXT: s_wait_loadcnt 0x08391; GFX12-NEXT: s_setpc_b64 s[30:31]8392;8393; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:8394; GFX942: ; %bb.0:8395; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8396; GFX942-NEXT: v_mov_b32_e32 v1, s168397; GFX942-NEXT: buffer_wbl2 sc18398; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:1024 sc08399; GFX942-NEXT: s_waitcnt vmcnt(0)8400; GFX942-NEXT: buffer_inv sc18401; GFX942-NEXT: s_setpc_b64 s[30:31]8402;8403; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:8404; GFX11: ; %bb.0:8405; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8406; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s168407; GFX11-NEXT: v_mov_b32_e32 v0, s168408; GFX11-NEXT: s_mov_b32 s4, 08409; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:10248410; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start8411; GFX11-NEXT: ; =>This Inner Loop Header: Depth=18412; GFX11-NEXT: s_waitcnt vmcnt(0)8413; GFX11-NEXT: v_mov_b32_e32 v5, v08414; GFX11-NEXT: s_waitcnt_vscnt null, 0x08415; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8416; GFX11-NEXT: v_pk_add_f16 v4, v5, v28417; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v58418; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc8419; GFX11-NEXT: s_waitcnt vmcnt(0)8420; GFX11-NEXT: buffer_gl1_inv8421; GFX11-NEXT: buffer_gl0_inv8422; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v58423; GFX11-NEXT: s_or_b32 s4, vcc_lo, s48424; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8425; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s48426; GFX11-NEXT: s_cbranch_execnz .LBB24_18427; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end8428; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s48429; GFX11-NEXT: s_setpc_b64 s[30:31]8430;8431; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:8432; GFX10: ; %bb.0:8433; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8434; GFX10-NEXT: v_mov_b32_e32 v2, v08435; GFX10-NEXT: v_mov_b32_e32 v0, s208436; GFX10-NEXT: v_mov_b32_e32 v3, s208437; GFX10-NEXT: s_mov_b32 s4, 08438; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10248439; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start8440; GFX10-NEXT: ; =>This Inner Loop Header: Depth=18441; GFX10-NEXT: s_waitcnt vmcnt(0)8442; GFX10-NEXT: v_mov_b32_e32 v5, v08443; GFX10-NEXT: s_waitcnt_vscnt null, 0x08444; GFX10-NEXT: v_pk_add_f16 v4, v5, v28445; GFX10-NEXT: v_mov_b32_e32 v0, v48446; GFX10-NEXT: v_mov_b32_e32 v1, v58447; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc8448; GFX10-NEXT: s_waitcnt vmcnt(0)8449; GFX10-NEXT: buffer_gl1_inv8450; GFX10-NEXT: buffer_gl0_inv8451; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v58452; GFX10-NEXT: s_or_b32 s4, vcc_lo, s48453; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s48454; GFX10-NEXT: s_cbranch_execnz .LBB24_18455; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end8456; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s48457; GFX10-NEXT: s_setpc_b64 s[30:31]8458;8459; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:8460; GFX90A: ; %bb.0:8461; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8462; GFX90A-NEXT: v_mov_b32_e32 v2, v08463; GFX90A-NEXT: v_mov_b32_e32 v0, s208464; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10248465; GFX90A-NEXT: s_mov_b64 s[4:5], 08466; GFX90A-NEXT: v_mov_b32_e32 v3, s208467; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start8468; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=18469; GFX90A-NEXT: s_waitcnt vmcnt(0)8470; GFX90A-NEXT: v_mov_b32_e32 v5, v08471; GFX90A-NEXT: v_pk_add_f16 v4, v5, v28472; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]8473; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc8474; GFX90A-NEXT: s_waitcnt vmcnt(0)8475; GFX90A-NEXT: buffer_wbinvl18476; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v58477; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8478; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]8479; GFX90A-NEXT: s_cbranch_execnz .LBB24_18480; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end8481; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]8482; GFX90A-NEXT: s_setpc_b64 s[30:31]8483;8484; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:8485; GFX908: ; %bb.0:8486; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8487; GFX908-NEXT: v_mov_b32_e32 v2, v08488; GFX908-NEXT: v_mov_b32_e32 v0, s208489; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10248490; GFX908-NEXT: s_mov_b64 s[4:5], 08491; GFX908-NEXT: v_mov_b32_e32 v3, s208492; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start8493; GFX908-NEXT: ; =>This Inner Loop Header: Depth=18494; GFX908-NEXT: s_waitcnt vmcnt(0)8495; GFX908-NEXT: v_mov_b32_e32 v5, v08496; GFX908-NEXT: v_pk_add_f16 v4, v5, v28497; GFX908-NEXT: v_mov_b32_e32 v0, v48498; GFX908-NEXT: v_mov_b32_e32 v1, v58499; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc8500; GFX908-NEXT: s_waitcnt vmcnt(0)8501; GFX908-NEXT: buffer_wbinvl18502; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v58503; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8504; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]8505; GFX908-NEXT: s_cbranch_execnz .LBB24_18506; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end8507; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]8508; GFX908-NEXT: s_setpc_b64 s[30:31]8509;8510; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:8511; GFX8: ; %bb.0:8512; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8513; GFX8-NEXT: v_mov_b32_e32 v2, v08514; GFX8-NEXT: v_mov_b32_e32 v0, s208515; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10248516; GFX8-NEXT: s_mov_b64 s[4:5], 08517; GFX8-NEXT: v_mov_b32_e32 v3, s208518; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start8519; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18520; GFX8-NEXT: s_waitcnt vmcnt(0)8521; GFX8-NEXT: v_mov_b32_e32 v5, v08522; GFX8-NEXT: v_add_f16_sdwa v0, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_18523; GFX8-NEXT: v_add_f16_e32 v1, v5, v28524; GFX8-NEXT: v_or_b32_e32 v4, v1, v08525; GFX8-NEXT: v_mov_b32_e32 v0, v48526; GFX8-NEXT: v_mov_b32_e32 v1, v58527; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc8528; GFX8-NEXT: s_waitcnt vmcnt(0)8529; GFX8-NEXT: buffer_wbinvl18530; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v58531; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8532; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]8533; GFX8-NEXT: s_cbranch_execnz .LBB24_18534; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8535; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]8536; GFX8-NEXT: s_setpc_b64 s[30:31]8537;8538; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:8539; GFX7: ; %bb.0:8540; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8541; GFX7-NEXT: v_mov_b32_e32 v2, s208542; GFX7-NEXT: buffer_load_dword v3, v2, s[16:19], 0 offen offset:10248543; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v18544; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v08545; GFX7-NEXT: s_mov_b64 s[4:5], 08546; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v18547; GFX7-NEXT: s_waitcnt vmcnt(0)8548; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v38549; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v38550; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v18551; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v48552; GFX7-NEXT: v_mov_b32_e32 v4, s208553; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start8554; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18555; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v18556; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v08557; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v18558; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v08559; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v18560; GFX7-NEXT: v_add_f32_e32 v5, v5, v28561; GFX7-NEXT: v_add_f32_e32 v6, v6, v38562; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v58563; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v68564; GFX7-NEXT: v_or_b32_e32 v6, v0, v18565; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v58566; GFX7-NEXT: v_or_b32_e32 v5, v7, v08567; GFX7-NEXT: v_mov_b32_e32 v8, v68568; GFX7-NEXT: v_mov_b32_e32 v7, v58569; GFX7-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[16:19], 0 offen offset:1024 glc8570; GFX7-NEXT: s_waitcnt vmcnt(0)8571; GFX7-NEXT: buffer_wbinvl18572; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v78573; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v78574; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v18575; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v7, v68576; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8577; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]8578; GFX7-NEXT: s_cbranch_execnz .LBB24_18579; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8580; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]8581; GFX7-NEXT: s_setpc_b64 s[30:31]8582;8583; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:8584; GFX6: ; %bb.0:8585; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8586; GFX6-NEXT: v_mov_b32_e32 v2, s208587; GFX6-NEXT: buffer_load_dword v3, v2, s[16:19], 0 offen offset:10248588; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v18589; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v08590; GFX6-NEXT: s_add_i32 s6, s20, 0x4008591; GFX6-NEXT: s_mov_b64 s[4:5], 08592; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v18593; GFX6-NEXT: s_waitcnt vmcnt(0)8594; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v38595; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v38596; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v18597; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v48598; GFX6-NEXT: v_mov_b32_e32 v4, s68599; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start8600; GFX6-NEXT: ; =>This Inner Loop Header: Depth=18601; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v18602; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v08603; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v18604; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v08605; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v18606; GFX6-NEXT: v_add_f32_e32 v5, v5, v28607; GFX6-NEXT: v_add_f32_e32 v6, v6, v38608; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v58609; GFX6-NEXT: s_waitcnt expcnt(0)8610; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v68611; GFX6-NEXT: v_or_b32_e32 v6, v0, v18612; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v58613; GFX6-NEXT: v_or_b32_e32 v5, v7, v08614; GFX6-NEXT: v_mov_b32_e32 v8, v68615; GFX6-NEXT: v_mov_b32_e32 v7, v58616; GFX6-NEXT: buffer_atomic_cmpswap v[7:8], v4, s[16:19], 0 offen glc8617; GFX6-NEXT: s_waitcnt vmcnt(0)8618; GFX6-NEXT: buffer_wbinvl18619; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v78620; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v78621; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v18622; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v7, v68623; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8624; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]8625; GFX6-NEXT: s_cbranch_execnz .LBB24_18626; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end8627; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]8628; GFX6-NEXT: s_waitcnt expcnt(0)8629; GFX6-NEXT: s_setpc_b64 s[30:31]8630 %gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 2568631 %result = atomicrmw fadd ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08632 ret <2 x half> %result8633}8634 8635define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, <2 x half> %val) #0 {8636; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:8637; GFX12: ; %bb.0:8638; GFX12-NEXT: s_wait_loadcnt_dscnt 0x08639; GFX12-NEXT: s_wait_expcnt 0x08640; GFX12-NEXT: s_wait_samplecnt 0x08641; GFX12-NEXT: s_wait_bvhcnt 0x08642; GFX12-NEXT: s_wait_kmcnt 0x08643; GFX12-NEXT: v_mov_b32_e32 v1, s168644; GFX12-NEXT: s_wait_storecnt 0x08645; GFX12-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], null offen offset:10248646; GFX12-NEXT: s_wait_storecnt 0x08647; GFX12-NEXT: global_inv scope:SCOPE_DEV8648; GFX12-NEXT: s_wait_loadcnt 0x08649; GFX12-NEXT: s_setpc_b64 s[30:31]8650;8651; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:8652; GFX942: ; %bb.0:8653; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8654; GFX942-NEXT: v_mov_b32_e32 v1, s168655; GFX942-NEXT: buffer_wbl2 sc18656; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v1, s[0:3], 0 offen offset:10248657; GFX942-NEXT: s_waitcnt vmcnt(0)8658; GFX942-NEXT: buffer_inv sc18659; GFX942-NEXT: s_setpc_b64 s[30:31]8660;8661; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:8662; GFX11: ; %bb.0:8663; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8664; GFX11-NEXT: v_mov_b32_e32 v1, s168665; GFX11-NEXT: v_mov_b32_e32 v3, s168666; GFX11-NEXT: s_mov_b32 s4, 08667; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen offset:10248668; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start8669; GFX11-NEXT: ; =>This Inner Loop Header: Depth=18670; GFX11-NEXT: s_waitcnt vmcnt(0)8671; GFX11-NEXT: v_pk_add_f16 v1, v2, v08672; GFX11-NEXT: v_mov_b32_e32 v5, v28673; GFX11-NEXT: s_waitcnt_vscnt null, 0x08674; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)8675; GFX11-NEXT: v_mov_b32_e32 v4, v18676; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc8677; GFX11-NEXT: s_waitcnt vmcnt(0)8678; GFX11-NEXT: buffer_gl1_inv8679; GFX11-NEXT: buffer_gl0_inv8680; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v28681; GFX11-NEXT: v_mov_b32_e32 v2, v48682; GFX11-NEXT: s_or_b32 s4, vcc_lo, s48683; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8684; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s48685; GFX11-NEXT: s_cbranch_execnz .LBB25_18686; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end8687; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s48688; GFX11-NEXT: s_setpc_b64 s[30:31]8689;8690; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:8691; GFX10: ; %bb.0:8692; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8693; GFX10-NEXT: v_mov_b32_e32 v1, s208694; GFX10-NEXT: v_mov_b32_e32 v3, s208695; GFX10-NEXT: s_mov_b32 s4, 08696; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10248697; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start8698; GFX10-NEXT: ; =>This Inner Loop Header: Depth=18699; GFX10-NEXT: s_waitcnt vmcnt(0)8700; GFX10-NEXT: v_pk_add_f16 v1, v2, v08701; GFX10-NEXT: v_mov_b32_e32 v5, v28702; GFX10-NEXT: s_waitcnt_vscnt null, 0x08703; GFX10-NEXT: v_mov_b32_e32 v4, v18704; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc8705; GFX10-NEXT: s_waitcnt vmcnt(0)8706; GFX10-NEXT: buffer_gl1_inv8707; GFX10-NEXT: buffer_gl0_inv8708; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v28709; GFX10-NEXT: v_mov_b32_e32 v2, v48710; GFX10-NEXT: s_or_b32 s4, vcc_lo, s48711; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s48712; GFX10-NEXT: s_cbranch_execnz .LBB25_18713; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end8714; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s48715; GFX10-NEXT: s_setpc_b64 s[30:31]8716;8717; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:8718; GFX90A: ; %bb.0:8719; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8720; GFX90A-NEXT: v_mov_b32_e32 v1, s208721; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen offset:10248722; GFX90A-NEXT: s_mov_b64 s[4:5], 08723; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start8724; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=18725; GFX90A-NEXT: s_waitcnt vmcnt(0)8726; GFX90A-NEXT: v_pk_add_f16 v2, v3, v08727; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]8728; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc8729; GFX90A-NEXT: s_waitcnt vmcnt(0)8730; GFX90A-NEXT: buffer_wbinvl18731; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v38732; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8733; GFX90A-NEXT: v_mov_b32_e32 v3, v48734; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]8735; GFX90A-NEXT: s_cbranch_execnz .LBB25_18736; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end8737; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]8738; GFX90A-NEXT: s_setpc_b64 s[30:31]8739;8740; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:8741; GFX908: ; %bb.0:8742; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8743; GFX908-NEXT: v_mov_b32_e32 v1, s208744; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10248745; GFX908-NEXT: s_mov_b64 s[4:5], 08746; GFX908-NEXT: v_mov_b32_e32 v3, s208747; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start8748; GFX908-NEXT: ; =>This Inner Loop Header: Depth=18749; GFX908-NEXT: s_waitcnt vmcnt(0)8750; GFX908-NEXT: v_pk_add_f16 v1, v2, v08751; GFX908-NEXT: v_mov_b32_e32 v5, v28752; GFX908-NEXT: v_mov_b32_e32 v4, v18753; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc8754; GFX908-NEXT: s_waitcnt vmcnt(0)8755; GFX908-NEXT: buffer_wbinvl18756; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v28757; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8758; GFX908-NEXT: v_mov_b32_e32 v2, v48759; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]8760; GFX908-NEXT: s_cbranch_execnz .LBB25_18761; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end8762; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]8763; GFX908-NEXT: s_setpc_b64 s[30:31]8764;8765; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:8766; GFX8: ; %bb.0:8767; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8768; GFX8-NEXT: v_mov_b32_e32 v1, s208769; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:10248770; GFX8-NEXT: s_mov_b64 s[4:5], 08771; GFX8-NEXT: v_mov_b32_e32 v3, s208772; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start8773; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18774; GFX8-NEXT: s_waitcnt vmcnt(0)8775; GFX8-NEXT: v_add_f16_sdwa v1, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_18776; GFX8-NEXT: v_add_f16_e32 v4, v2, v08777; GFX8-NEXT: v_or_b32_e32 v1, v4, v18778; GFX8-NEXT: v_mov_b32_e32 v5, v28779; GFX8-NEXT: v_mov_b32_e32 v4, v18780; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc8781; GFX8-NEXT: s_waitcnt vmcnt(0)8782; GFX8-NEXT: buffer_wbinvl18783; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v28784; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8785; GFX8-NEXT: v_mov_b32_e32 v2, v48786; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]8787; GFX8-NEXT: s_cbranch_execnz .LBB25_18788; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8789; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]8790; GFX8-NEXT: s_setpc_b64 s[30:31]8791;8792; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:8793; GFX7: ; %bb.0:8794; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8795; GFX7-NEXT: v_mov_b32_e32 v2, s208796; GFX7-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:10248797; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v18798; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v08799; GFX7-NEXT: s_mov_b64 s[4:5], 08800; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v18801; GFX7-NEXT: s_waitcnt vmcnt(0)8802; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v28803; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v28804; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v18805; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v58806; GFX7-NEXT: v_mov_b32_e32 v2, s208807; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start8808; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18809; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v48810; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v38811; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v48812; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v38813; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v48814; GFX7-NEXT: v_add_f32_e32 v5, v5, v08815; GFX7-NEXT: v_add_f32_e32 v6, v6, v18816; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v58817; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v68818; GFX7-NEXT: v_or_b32_e32 v5, v3, v48819; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v78820; GFX7-NEXT: v_or_b32_e32 v4, v6, v38821; GFX7-NEXT: v_mov_b32_e32 v7, v58822; GFX7-NEXT: v_mov_b32_e32 v6, v48823; GFX7-NEXT: buffer_atomic_cmpswap v[6:7], v2, s[16:19], 0 offen offset:1024 glc8824; GFX7-NEXT: s_waitcnt vmcnt(0)8825; GFX7-NEXT: buffer_wbinvl18826; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v68827; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v68828; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v48829; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v6, v58830; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8831; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]8832; GFX7-NEXT: s_cbranch_execnz .LBB25_18833; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8834; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]8835; GFX7-NEXT: s_setpc_b64 s[30:31]8836;8837; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_remote_memory:8838; GFX6: ; %bb.0:8839; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8840; GFX6-NEXT: v_mov_b32_e32 v2, s208841; GFX6-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:10248842; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v18843; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v08844; GFX6-NEXT: s_add_i32 s6, s20, 0x4008845; GFX6-NEXT: s_mov_b64 s[4:5], 08846; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v18847; GFX6-NEXT: s_waitcnt vmcnt(0)8848; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v28849; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v28850; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v18851; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v58852; GFX6-NEXT: v_mov_b32_e32 v2, s68853; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start8854; GFX6-NEXT: ; =>This Inner Loop Header: Depth=18855; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v48856; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v38857; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v48858; GFX6-NEXT: s_waitcnt expcnt(0)8859; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v38860; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v48861; GFX6-NEXT: v_add_f32_e32 v5, v5, v08862; GFX6-NEXT: v_add_f32_e32 v6, v6, v18863; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v58864; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v68865; GFX6-NEXT: v_or_b32_e32 v5, v3, v48866; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v78867; GFX6-NEXT: v_or_b32_e32 v4, v6, v38868; GFX6-NEXT: v_mov_b32_e32 v7, v58869; GFX6-NEXT: v_mov_b32_e32 v6, v48870; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v2, s[16:19], 0 offen glc8871; GFX6-NEXT: s_waitcnt vmcnt(0)8872; GFX6-NEXT: buffer_wbinvl18873; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v68874; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v68875; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v48876; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v58877; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8878; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]8879; GFX6-NEXT: s_cbranch_execnz .LBB25_18880; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end8881; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]8882; GFX6-NEXT: s_waitcnt expcnt(0)8883; GFX6-NEXT: s_setpc_b64 s[30:31]8884 %gep = getelementptr <2 x half>, ptr addrspace(7) %ptr, i32 2568885 %unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x half> %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !08886 ret void8887}8888 8889; --------------------------------------------------------------------8890; <2 x bfloat>8891; --------------------------------------------------------------------8892 8893define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {8894; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:8895; GFX12: ; %bb.0:8896; GFX12-NEXT: s_wait_loadcnt_dscnt 0x08897; GFX12-NEXT: s_wait_expcnt 0x08898; GFX12-NEXT: s_wait_samplecnt 0x08899; GFX12-NEXT: s_wait_bvhcnt 0x08900; GFX12-NEXT: s_wait_kmcnt 0x08901; GFX12-NEXT: v_mov_b32_e32 v1, s168902; GFX12-NEXT: s_wait_storecnt 0x08903; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN8904; GFX12-NEXT: s_wait_loadcnt 0x08905; GFX12-NEXT: global_inv scope:SCOPE_DEV8906; GFX12-NEXT: s_wait_loadcnt 0x08907; GFX12-NEXT: s_setpc_b64 s[30:31]8908;8909; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:8910; GFX942: ; %bb.0:8911; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8912; GFX942-NEXT: v_mov_b32_e32 v1, v08913; GFX942-NEXT: v_mov_b32_e32 v0, s168914; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:10248915; GFX942-NEXT: s_mov_b64 s[6:7], 08916; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v18917; GFX942-NEXT: s_movk_i32 s8, 0x7fff8918; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v18919; GFX942-NEXT: s_mov_b32 s9, 0x70603028920; GFX942-NEXT: v_mov_b32_e32 v4, s168921; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start8922; GFX942-NEXT: ; =>This Inner Loop Header: Depth=18923; GFX942-NEXT: s_waitcnt vmcnt(0)8924; GFX942-NEXT: v_mov_b32_e32 v7, v08925; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v78926; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v78927; GFX942-NEXT: v_add_f32_e32 v0, v0, v28928; GFX942-NEXT: v_add_f32_e32 v1, v1, v38929; GFX942-NEXT: v_bfe_u32 v5, v0, 16, 18930; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 18931; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v08932; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v18933; GFX942-NEXT: v_add3_u32 v5, v5, v0, s88934; GFX942-NEXT: v_add3_u32 v8, v8, v1, s88935; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v18936; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v08937; GFX942-NEXT: buffer_wbl2 sc18938; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc8939; GFX942-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]8940; GFX942-NEXT: v_perm_b32 v6, v1, v0, s98941; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[6:7]8942; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[0:3], 0 offen offset:1024 sc08943; GFX942-NEXT: s_waitcnt vmcnt(0)8944; GFX942-NEXT: buffer_inv sc18945; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v78946; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]8947; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]8948; GFX942-NEXT: s_cbranch_execnz .LBB26_18949; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end8950; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]8951; GFX942-NEXT: s_setpc_b64 s[30:31]8952;8953; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:8954; GFX11-TRUE16: ; %bb.0:8955; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8956; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s168957; GFX11-TRUE16-NEXT: s_mov_b32 s4, 08958; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)8959; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v18960; GFX11-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:10248961; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v18962; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x18963; GFX11-TRUE16-NEXT: .p2align 68964; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start8965; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=18966; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)8967; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v08968; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x08969; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8970; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v68971; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v28972; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)8973; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 18974; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v18975; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff8976; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v68977; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8978; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v38979; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 18980; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v08981; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v08982; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)8983; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff8984; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo8985; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v18986; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo8987; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)8988; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h8989; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v68990; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc8991; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)8992; GFX11-TRUE16-NEXT: buffer_gl1_inv8993; GFX11-TRUE16-NEXT: buffer_gl0_inv8994; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v68995; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s48996; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8997; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s48998; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_18999; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end9000; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x29001; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s49002; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]9003;9004; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:9005; GFX11-FAKE16: ; %bb.0:9006; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9007; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s169008; GFX11-FAKE16-NEXT: s_mov_b32 s5, 09009; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9010; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v19011; GFX11-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:10249012; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v19013; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x19014; GFX11-FAKE16-NEXT: .p2align 69015; GFX11-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start9016; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=19017; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)9018; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v09019; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x09020; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9021; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v69022; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v39023; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)9024; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 19025; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v19026; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v19027; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff9028; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9029; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v69030; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v29031; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)9032; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 19033; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v09034; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v09035; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff9036; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9037; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s49038; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x70603029039; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)9040; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v69041; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc9042; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)9043; GFX11-FAKE16-NEXT: buffer_gl1_inv9044; GFX11-FAKE16-NEXT: buffer_gl0_inv9045; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v69046; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s59047; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9048; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s59049; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB26_19050; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end9051; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x29052; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s59053; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]9054;9055; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:9056; GFX10: ; %bb.0:9057; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9058; GFX10-NEXT: v_mov_b32_e32 v1, v09059; GFX10-NEXT: v_mov_b32_e32 v0, s209060; GFX10-NEXT: v_mov_b32_e32 v4, s209061; GFX10-NEXT: s_mov_b32 s5, 09062; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v19063; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10249064; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v19065; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start9066; GFX10-NEXT: ; =>This Inner Loop Header: Depth=19067; GFX10-NEXT: s_waitcnt vmcnt(0)9068; GFX10-NEXT: v_mov_b32_e32 v6, v09069; GFX10-NEXT: s_waitcnt_vscnt null, 0x09070; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v69071; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v69072; GFX10-NEXT: v_add_f32_e32 v0, v0, v29073; GFX10-NEXT: v_add_f32_e32 v1, v1, v39074; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 19075; GFX10-NEXT: v_bfe_u32 v7, v1, 16, 19076; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v09077; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v19078; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v19079; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff9080; GFX10-NEXT: v_add3_u32 v7, v7, v1, 0x7fff9081; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v09082; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo9083; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v8, s49084; GFX10-NEXT: v_perm_b32 v5, v1, v0, 0x70603029085; GFX10-NEXT: v_mov_b32_e32 v0, v59086; GFX10-NEXT: v_mov_b32_e32 v1, v69087; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc9088; GFX10-NEXT: s_waitcnt vmcnt(0)9089; GFX10-NEXT: buffer_gl1_inv9090; GFX10-NEXT: buffer_gl0_inv9091; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v69092; GFX10-NEXT: s_or_b32 s5, vcc_lo, s59093; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s59094; GFX10-NEXT: s_cbranch_execnz .LBB26_19095; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end9096; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s59097; GFX10-NEXT: s_setpc_b64 s[30:31]9098;9099; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:9100; GFX90A: ; %bb.0:9101; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9102; GFX90A-NEXT: v_mov_b32_e32 v1, v09103; GFX90A-NEXT: v_mov_b32_e32 v0, s209104; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10249105; GFX90A-NEXT: s_mov_b64 s[6:7], 09106; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v19107; GFX90A-NEXT: s_movk_i32 s8, 0x7fff9108; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v19109; GFX90A-NEXT: s_mov_b32 s9, 0x70603029110; GFX90A-NEXT: v_mov_b32_e32 v4, s209111; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start9112; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19113; GFX90A-NEXT: s_waitcnt vmcnt(0)9114; GFX90A-NEXT: v_mov_b32_e32 v7, v09115; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v79116; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v79117; GFX90A-NEXT: v_add_f32_e32 v0, v0, v29118; GFX90A-NEXT: v_add_f32_e32 v1, v1, v39119; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 19120; GFX90A-NEXT: v_bfe_u32 v8, v1, 16, 19121; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v09122; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v19123; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s89124; GFX90A-NEXT: v_add3_u32 v8, v8, v1, s89125; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v19126; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v09127; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]9128; GFX90A-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc9129; GFX90A-NEXT: v_perm_b32 v6, v1, v0, s99130; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]9131; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc9132; GFX90A-NEXT: s_waitcnt vmcnt(0)9133; GFX90A-NEXT: buffer_wbinvl19134; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v79135; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]9136; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]9137; GFX90A-NEXT: s_cbranch_execnz .LBB26_19138; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9139; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]9140; GFX90A-NEXT: s_setpc_b64 s[30:31]9141;9142; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:9143; GFX908: ; %bb.0:9144; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9145; GFX908-NEXT: v_mov_b32_e32 v1, v09146; GFX908-NEXT: v_mov_b32_e32 v0, s209147; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10249148; GFX908-NEXT: s_mov_b64 s[6:7], 09149; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v19150; GFX908-NEXT: s_movk_i32 s8, 0x7fff9151; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v19152; GFX908-NEXT: s_mov_b32 s9, 0x70603029153; GFX908-NEXT: v_mov_b32_e32 v4, s209154; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start9155; GFX908-NEXT: ; =>This Inner Loop Header: Depth=19156; GFX908-NEXT: s_waitcnt vmcnt(0)9157; GFX908-NEXT: v_mov_b32_e32 v6, v09158; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v69159; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v69160; GFX908-NEXT: v_add_f32_e32 v0, v0, v29161; GFX908-NEXT: v_add_f32_e32 v1, v1, v39162; GFX908-NEXT: v_bfe_u32 v5, v0, 16, 19163; GFX908-NEXT: v_bfe_u32 v8, v1, 16, 19164; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v09165; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v19166; GFX908-NEXT: v_add3_u32 v5, v5, v0, s89167; GFX908-NEXT: v_add3_u32 v8, v8, v1, s89168; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v19169; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v09170; GFX908-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]9171; GFX908-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc9172; GFX908-NEXT: v_perm_b32 v5, v1, v0, s99173; GFX908-NEXT: v_mov_b32_e32 v0, v59174; GFX908-NEXT: v_mov_b32_e32 v1, v69175; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc9176; GFX908-NEXT: s_waitcnt vmcnt(0)9177; GFX908-NEXT: buffer_wbinvl19178; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v69179; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]9180; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]9181; GFX908-NEXT: s_cbranch_execnz .LBB26_19182; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end9183; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]9184; GFX908-NEXT: s_setpc_b64 s[30:31]9185;9186; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:9187; GFX8: ; %bb.0:9188; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9189; GFX8-NEXT: v_mov_b32_e32 v1, v09190; GFX8-NEXT: v_mov_b32_e32 v0, s209191; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:10249192; GFX8-NEXT: s_mov_b64 s[6:7], 09193; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v19194; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v19195; GFX8-NEXT: v_mov_b32_e32 v4, s209196; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start9197; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19198; GFX8-NEXT: s_waitcnt vmcnt(0)9199; GFX8-NEXT: v_mov_b32_e32 v6, v09200; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v69201; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v69202; GFX8-NEXT: v_add_f32_e32 v0, v0, v29203; GFX8-NEXT: v_add_f32_e32 v1, v1, v39204; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 19205; GFX8-NEXT: v_bfe_u32 v8, v1, 16, 19206; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v09207; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v19208; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v59209; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v89210; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v19211; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v19212; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v09213; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v09214; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc9215; GFX8-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]9216; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v19217; GFX8-NEXT: v_alignbit_b32 v5, v1, v0, 169218; GFX8-NEXT: v_mov_b32_e32 v0, v59219; GFX8-NEXT: v_mov_b32_e32 v1, v69220; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc9221; GFX8-NEXT: s_waitcnt vmcnt(0)9222; GFX8-NEXT: buffer_wbinvl19223; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v69224; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]9225; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]9226; GFX8-NEXT: s_cbranch_execnz .LBB26_19227; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9228; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]9229; GFX8-NEXT: s_setpc_b64 s[30:31]9230;9231; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:9232; GFX7: ; %bb.0:9233; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9234; GFX7-NEXT: v_mov_b32_e32 v2, s209235; GFX7-NEXT: buffer_load_dword v4, v2, s[16:19], 0 offen offset:10249236; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v09237; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v19238; GFX7-NEXT: s_mov_b64 s[4:5], 09239; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v09240; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v19241; GFX7-NEXT: s_waitcnt vmcnt(0)9242; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v49243; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v49244; GFX7-NEXT: v_mov_b32_e32 v4, s209245; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start9246; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19247; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v19248; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v09249; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v19250; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v09251; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v19252; GFX7-NEXT: v_add_f32_e32 v6, v6, v39253; GFX7-NEXT: v_add_f32_e32 v5, v5, v29254; GFX7-NEXT: v_alignbit_b32 v1, v1, v0, 169255; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v69256; GFX7-NEXT: v_alignbit_b32 v0, v0, v5, 169257; GFX7-NEXT: v_mov_b32_e32 v6, v19258; GFX7-NEXT: v_mov_b32_e32 v5, v09259; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc9260; GFX7-NEXT: s_waitcnt vmcnt(0)9261; GFX7-NEXT: buffer_wbinvl19262; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v19263; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v59264; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9265; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v59266; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]9267; GFX7-NEXT: s_cbranch_execnz .LBB26_19268; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9269; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]9270; GFX7-NEXT: s_setpc_b64 s[30:31]9271;9272; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:9273; GFX6: ; %bb.0:9274; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9275; GFX6-NEXT: v_mov_b32_e32 v2, s209276; GFX6-NEXT: buffer_load_dword v4, v2, s[16:19], 0 offen offset:10249277; GFX6-NEXT: s_add_i32 s6, s20, 0x4009278; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v09279; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v19280; GFX6-NEXT: s_mov_b64 s[4:5], 09281; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v09282; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v19283; GFX6-NEXT: s_waitcnt vmcnt(0)9284; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v49285; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v49286; GFX6-NEXT: v_mov_b32_e32 v4, s69287; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start9288; GFX6-NEXT: ; =>This Inner Loop Header: Depth=19289; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v19290; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v09291; GFX6-NEXT: s_waitcnt expcnt(0)9292; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v19293; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v09294; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v19295; GFX6-NEXT: v_add_f32_e32 v6, v6, v39296; GFX6-NEXT: v_add_f32_e32 v5, v5, v29297; GFX6-NEXT: v_alignbit_b32 v1, v1, v0, 169298; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v69299; GFX6-NEXT: v_alignbit_b32 v0, v0, v5, 169300; GFX6-NEXT: v_mov_b32_e32 v6, v19301; GFX6-NEXT: v_mov_b32_e32 v5, v09302; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen glc9303; GFX6-NEXT: s_waitcnt vmcnt(0)9304; GFX6-NEXT: buffer_wbinvl19305; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v19306; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v59307; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9308; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v59309; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]9310; GFX6-NEXT: s_cbranch_execnz .LBB26_19311; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end9312; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]9313; GFX6-NEXT: s_waitcnt expcnt(0)9314; GFX6-NEXT: s_setpc_b64 s[30:31]9315 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 2569316 %result = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09317 ret <2 x bfloat> %result9318}9319 9320define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {9321; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:9322; GFX12: ; %bb.0:9323; GFX12-NEXT: s_wait_loadcnt_dscnt 0x09324; GFX12-NEXT: s_wait_expcnt 0x09325; GFX12-NEXT: s_wait_samplecnt 0x09326; GFX12-NEXT: s_wait_bvhcnt 0x09327; GFX12-NEXT: s_wait_kmcnt 0x09328; GFX12-NEXT: v_mov_b32_e32 v1, s169329; GFX12-NEXT: s_wait_storecnt 0x09330; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:10249331; GFX12-NEXT: s_wait_storecnt 0x09332; GFX12-NEXT: global_inv scope:SCOPE_DEV9333; GFX12-NEXT: s_wait_loadcnt 0x09334; GFX12-NEXT: s_setpc_b64 s[30:31]9335;9336; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:9337; GFX942: ; %bb.0:9338; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9339; GFX942-NEXT: v_mov_b32_e32 v1, s169340; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:10249341; GFX942-NEXT: s_mov_b64 s[6:7], 09342; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v09343; GFX942-NEXT: s_movk_i32 s8, 0x7fff9344; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v09345; GFX942-NEXT: s_mov_b32 s9, 0x70603029346; GFX942-NEXT: v_mov_b32_e32 v4, s169347; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start9348; GFX942-NEXT: ; =>This Inner Loop Header: Depth=19349; GFX942-NEXT: s_waitcnt vmcnt(0)9350; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v19351; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v19352; GFX942-NEXT: v_add_f32_e32 v0, v0, v29353; GFX942-NEXT: v_add_f32_e32 v5, v5, v39354; GFX942-NEXT: v_bfe_u32 v6, v0, 16, 19355; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 19356; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v09357; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v59358; GFX942-NEXT: v_add3_u32 v6, v6, v0, s89359; GFX942-NEXT: v_add3_u32 v8, v8, v5, s89360; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v59361; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v09362; GFX942-NEXT: buffer_wbl2 sc19363; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc9364; GFX942-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]9365; GFX942-NEXT: v_perm_b32 v0, v5, v0, s99366; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1]9367; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen offset:1024 sc09368; GFX942-NEXT: s_waitcnt vmcnt(0)9369; GFX942-NEXT: buffer_inv sc19370; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v19371; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]9372; GFX942-NEXT: v_mov_b32_e32 v1, v69373; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]9374; GFX942-NEXT: s_cbranch_execnz .LBB27_19375; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end9376; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]9377; GFX942-NEXT: s_setpc_b64 s[30:31]9378;9379; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:9380; GFX11-TRUE16: ; %bb.0:9381; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9382; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v09383; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v09384; GFX11-TRUE16-NEXT: s_mov_b32 s4, 09385; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:10249386; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x19387; GFX11-TRUE16-NEXT: .p2align 69388; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start9389; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=19390; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)9391; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v19392; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v19393; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x09394; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9395; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v2 :: v_dual_add_f32 v0, v0, v39396; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 19397; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)9398; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 19399; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v09400; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v09401; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v59402; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff9403; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff9404; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)9405; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo9406; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v59407; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo9408; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)9409; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h9410; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v09411; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc9412; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)9413; GFX11-TRUE16-NEXT: buffer_gl1_inv9414; GFX11-TRUE16-NEXT: buffer_gl0_inv9415; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v19416; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v59417; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s49418; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9419; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s49420; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_19421; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end9422; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x29423; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s49424; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]9425;9426; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:9427; GFX11-FAKE16: ; %bb.0:9428; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9429; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v09430; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v09431; GFX11-FAKE16-NEXT: s_mov_b32 s5, 09432; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:10249433; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x19434; GFX11-FAKE16-NEXT: .p2align 69435; GFX11-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start9436; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=19437; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)9438; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v19439; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v19440; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x09441; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9442; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v29443; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 19444; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)9445; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 19446; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v09447; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v59448; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v59449; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff9450; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff9451; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v09452; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)9453; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo9454; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s49455; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9456; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x70603029457; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v09458; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc9459; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)9460; GFX11-FAKE16-NEXT: buffer_gl1_inv9461; GFX11-FAKE16-NEXT: buffer_gl0_inv9462; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v19463; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v59464; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s59465; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9466; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s59467; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB27_19468; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end9469; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x29470; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s59471; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]9472;9473; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:9474; GFX10: ; %bb.0:9475; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9476; GFX10-NEXT: v_mov_b32_e32 v1, s209477; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v09478; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v09479; GFX10-NEXT: v_mov_b32_e32 v4, s209480; GFX10-NEXT: s_mov_b32 s5, 09481; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:10249482; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start9483; GFX10-NEXT: ; =>This Inner Loop Header: Depth=19484; GFX10-NEXT: s_waitcnt vmcnt(0)9485; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v19486; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v19487; GFX10-NEXT: s_waitcnt_vscnt null, 0x09488; GFX10-NEXT: v_add_f32_e32 v0, v0, v29489; GFX10-NEXT: v_add_f32_e32 v5, v5, v39490; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 19491; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 19492; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v09493; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v59494; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v59495; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff9496; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff9497; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v09498; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo9499; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s49500; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x70603029501; GFX10-NEXT: v_mov_b32_e32 v6, v19502; GFX10-NEXT: v_mov_b32_e32 v5, v09503; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc9504; GFX10-NEXT: s_waitcnt vmcnt(0)9505; GFX10-NEXT: buffer_gl1_inv9506; GFX10-NEXT: buffer_gl0_inv9507; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v19508; GFX10-NEXT: v_mov_b32_e32 v1, v59509; GFX10-NEXT: s_or_b32 s5, vcc_lo, s59510; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s59511; GFX10-NEXT: s_cbranch_execnz .LBB27_19512; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end9513; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s59514; GFX10-NEXT: s_setpc_b64 s[30:31]9515;9516; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:9517; GFX90A: ; %bb.0:9518; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9519; GFX90A-NEXT: v_mov_b32_e32 v1, s209520; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:10249521; GFX90A-NEXT: s_mov_b64 s[6:7], 09522; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v09523; GFX90A-NEXT: s_movk_i32 s8, 0x7fff9524; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v09525; GFX90A-NEXT: s_mov_b32 s9, 0x70603029526; GFX90A-NEXT: v_mov_b32_e32 v4, s209527; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start9528; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19529; GFX90A-NEXT: s_waitcnt vmcnt(0)9530; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v19531; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v19532; GFX90A-NEXT: v_add_f32_e32 v0, v0, v29533; GFX90A-NEXT: v_add_f32_e32 v5, v5, v39534; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 19535; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 19536; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v09537; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v59538; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s89539; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s89540; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v59541; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v09542; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]9543; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc9544; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s99545; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]9546; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen offset:1024 glc9547; GFX90A-NEXT: s_waitcnt vmcnt(0)9548; GFX90A-NEXT: buffer_wbinvl19549; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v19550; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]9551; GFX90A-NEXT: v_mov_b32_e32 v1, v69552; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]9553; GFX90A-NEXT: s_cbranch_execnz .LBB27_19554; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end9555; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]9556; GFX90A-NEXT: s_setpc_b64 s[30:31]9557;9558; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:9559; GFX908: ; %bb.0:9560; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9561; GFX908-NEXT: v_mov_b32_e32 v1, s209562; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:10249563; GFX908-NEXT: s_mov_b64 s[6:7], 09564; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v09565; GFX908-NEXT: s_movk_i32 s8, 0x7fff9566; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v09567; GFX908-NEXT: s_mov_b32 s9, 0x70603029568; GFX908-NEXT: v_mov_b32_e32 v4, s209569; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start9570; GFX908-NEXT: ; =>This Inner Loop Header: Depth=19571; GFX908-NEXT: s_waitcnt vmcnt(0)9572; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v19573; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v19574; GFX908-NEXT: v_add_f32_e32 v0, v0, v29575; GFX908-NEXT: v_add_f32_e32 v5, v5, v39576; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 19577; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 19578; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v09579; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v59580; GFX908-NEXT: v_add3_u32 v6, v6, v0, s89581; GFX908-NEXT: v_add3_u32 v8, v8, v5, s89582; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v59583; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v09584; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]9585; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc9586; GFX908-NEXT: v_perm_b32 v0, v5, v0, s99587; GFX908-NEXT: v_mov_b32_e32 v6, v19588; GFX908-NEXT: v_mov_b32_e32 v5, v09589; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc9590; GFX908-NEXT: s_waitcnt vmcnt(0)9591; GFX908-NEXT: buffer_wbinvl19592; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v19593; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]9594; GFX908-NEXT: v_mov_b32_e32 v1, v59595; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]9596; GFX908-NEXT: s_cbranch_execnz .LBB27_19597; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end9598; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]9599; GFX908-NEXT: s_setpc_b64 s[30:31]9600;9601; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:9602; GFX8: ; %bb.0:9603; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9604; GFX8-NEXT: v_mov_b32_e32 v1, s209605; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:10249606; GFX8-NEXT: s_mov_b64 s[6:7], 09607; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v09608; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v09609; GFX8-NEXT: v_mov_b32_e32 v4, s209610; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start9611; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19612; GFX8-NEXT: s_waitcnt vmcnt(0)9613; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v19614; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v19615; GFX8-NEXT: v_add_f32_e32 v0, v0, v29616; GFX8-NEXT: v_add_f32_e32 v5, v5, v39617; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 19618; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 19619; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v09620; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v59621; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v69622; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v89623; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v59624; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v59625; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v09626; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v09627; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc9628; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]9629; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v59630; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 169631; GFX8-NEXT: v_mov_b32_e32 v6, v19632; GFX8-NEXT: v_mov_b32_e32 v5, v09633; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc9634; GFX8-NEXT: s_waitcnt vmcnt(0)9635; GFX8-NEXT: buffer_wbinvl19636; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v19637; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]9638; GFX8-NEXT: v_mov_b32_e32 v1, v59639; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]9640; GFX8-NEXT: s_cbranch_execnz .LBB27_19641; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end9642; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]9643; GFX8-NEXT: s_setpc_b64 s[30:31]9644;9645; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:9646; GFX7: ; %bb.0:9647; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9648; GFX7-NEXT: v_mov_b32_e32 v2, s209649; GFX7-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:10249650; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v09651; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v19652; GFX7-NEXT: s_mov_b64 s[4:5], 09653; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v09654; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v19655; GFX7-NEXT: s_waitcnt vmcnt(0)9656; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v29657; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v29658; GFX7-NEXT: v_mov_b32_e32 v2, s209659; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start9660; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19661; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v39662; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v49663; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v39664; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v49665; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v39666; GFX7-NEXT: v_add_f32_e32 v6, v6, v19667; GFX7-NEXT: v_add_f32_e32 v5, v5, v09668; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 169669; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v69670; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 169671; GFX7-NEXT: v_mov_b32_e32 v6, v49672; GFX7-NEXT: v_mov_b32_e32 v5, v39673; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc9674; GFX7-NEXT: s_waitcnt vmcnt(0)9675; GFX7-NEXT: buffer_wbinvl19676; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v49677; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v59678; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9679; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v59680; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]9681; GFX7-NEXT: s_cbranch_execnz .LBB27_19682; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9683; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]9684; GFX7-NEXT: s_setpc_b64 s[30:31]9685;9686; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:9687; GFX6: ; %bb.0:9688; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9689; GFX6-NEXT: v_mov_b32_e32 v2, s209690; GFX6-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:10249691; GFX6-NEXT: s_add_i32 s6, s20, 0x4009692; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v09693; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v19694; GFX6-NEXT: s_mov_b64 s[4:5], 09695; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v09696; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v19697; GFX6-NEXT: s_waitcnt vmcnt(0)9698; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v29699; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v29700; GFX6-NEXT: v_mov_b32_e32 v2, s69701; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start9702; GFX6-NEXT: ; =>This Inner Loop Header: Depth=19703; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v39704; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v49705; GFX6-NEXT: s_waitcnt expcnt(0)9706; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v39707; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v49708; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v39709; GFX6-NEXT: v_add_f32_e32 v6, v6, v19710; GFX6-NEXT: v_add_f32_e32 v5, v5, v09711; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 169712; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v69713; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 169714; GFX6-NEXT: v_mov_b32_e32 v6, v49715; GFX6-NEXT: v_mov_b32_e32 v5, v39716; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc9717; GFX6-NEXT: s_waitcnt vmcnt(0)9718; GFX6-NEXT: buffer_wbinvl19719; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v49720; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v59721; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9722; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v59723; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]9724; GFX6-NEXT: s_cbranch_execnz .LBB27_19725; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end9726; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]9727; GFX6-NEXT: s_waitcnt expcnt(0)9728; GFX6-NEXT: s_setpc_b64 s[30:31]9729 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 2569730 %unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !09731 ret void9732}9733 9734define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, <2 x bfloat> %val) #0 {9735; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:9736; GFX12: ; %bb.0:9737; GFX12-NEXT: s_wait_loadcnt_dscnt 0x09738; GFX12-NEXT: s_wait_expcnt 0x09739; GFX12-NEXT: s_wait_samplecnt 0x09740; GFX12-NEXT: s_wait_bvhcnt 0x09741; GFX12-NEXT: s_wait_kmcnt 0x09742; GFX12-NEXT: s_mov_b32 s1, exec_lo9743; GFX12-NEXT: s_wait_storecnt 0x09744; GFX12-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=19745; GFX12-NEXT: v_readfirstlane_b32 s4, v09746; GFX12-NEXT: v_readfirstlane_b32 s5, v19747; GFX12-NEXT: v_readfirstlane_b32 s6, v29748; GFX12-NEXT: v_readfirstlane_b32 s7, v39749; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)9750; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)9751; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]9752; GFX12-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]9753; GFX12-NEXT: s_and_b32 s0, vcc_lo, s09754; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)9755; GFX12-NEXT: s_and_saveexec_b32 s0, s09756; GFX12-NEXT: s_wait_loadcnt 0x09757; GFX12-NEXT: buffer_atomic_pk_add_bf16 v5, v4, s[4:7], null offen offset:1024 th:TH_ATOMIC_RETURN9758; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr39759; GFX12-NEXT: ; implicit-def: $vgpr49760; GFX12-NEXT: s_xor_b32 exec_lo, exec_lo, s09761; GFX12-NEXT: s_cbranch_execnz .LBB28_19762; GFX12-NEXT: ; %bb.2:9763; GFX12-NEXT: s_mov_b32 exec_lo, s19764; GFX12-NEXT: s_wait_loadcnt 0x09765; GFX12-NEXT: v_mov_b32_e32 v0, v59766; GFX12-NEXT: global_inv scope:SCOPE_DEV9767; GFX12-NEXT: s_wait_loadcnt 0x09768; GFX12-NEXT: s_setpc_b64 s[30:31]9769;9770; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:9771; GFX942: ; %bb.0:9772; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9773; GFX942-NEXT: s_mov_b64 s[2:3], exec9774; GFX942-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=19775; GFX942-NEXT: v_readfirstlane_b32 s4, v09776; GFX942-NEXT: v_readfirstlane_b32 s5, v19777; GFX942-NEXT: v_readfirstlane_b32 s6, v29778; GFX942-NEXT: v_readfirstlane_b32 s7, v39779; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]9780; GFX942-NEXT: s_nop 09781; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]9782; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]9783; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]9784; GFX942-NEXT: buffer_load_dword v9, v4, s[4:7], 0 offen offset:10249785; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]9786; GFX942-NEXT: s_cbranch_execnz .LBB28_19787; GFX942-NEXT: ; %bb.2:9788; GFX942-NEXT: s_mov_b64 exec, s[2:3]9789; GFX942-NEXT: s_mov_b64 s[2:3], 09790; GFX942-NEXT: v_lshlrev_b32_e32 v10, 16, v59791; GFX942-NEXT: s_movk_i32 s10, 0x7fff9792; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v59793; GFX942-NEXT: s_mov_b32 s11, 0x70603029794; GFX942-NEXT: .LBB28_3: ; %atomicrmw.start9795; GFX942-NEXT: ; =>This Loop Header: Depth=19796; GFX942-NEXT: ; Child Loop BB28_4 Depth 29797; GFX942-NEXT: s_waitcnt vmcnt(0)9798; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v99799; GFX942-NEXT: v_add_f32_e32 v6, v6, v109800; GFX942-NEXT: v_bfe_u32 v7, v6, 16, 19801; GFX942-NEXT: v_add3_u32 v7, v7, v6, s109802; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v69803; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v69804; GFX942-NEXT: s_mov_b64 s[8:9], exec9805; GFX942-NEXT: buffer_wbl2 sc19806; GFX942-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc9807; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v99808; GFX942-NEXT: v_add_f32_e32 v7, v7, v59809; GFX942-NEXT: v_bfe_u32 v8, v7, 16, 19810; GFX942-NEXT: v_add3_u32 v8, v8, v7, s109811; GFX942-NEXT: v_or_b32_e32 v11, 0x400000, v79812; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v7, v79813; GFX942-NEXT: s_nop 19814; GFX942-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc9815; GFX942-NEXT: v_perm_b32 v8, v7, v6, s119816; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]9817; GFX942-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=19818; GFX942-NEXT: ; => This Inner Loop Header: Depth=29819; GFX942-NEXT: v_readfirstlane_b32 s4, v09820; GFX942-NEXT: v_readfirstlane_b32 s5, v19821; GFX942-NEXT: v_readfirstlane_b32 s6, v29822; GFX942-NEXT: v_readfirstlane_b32 s7, v39823; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[0:1]9824; GFX942-NEXT: s_nop 09825; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]9826; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]9827; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]9828; GFX942-NEXT: s_waitcnt vmcnt(0)9829; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 sc09830; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]9831; GFX942-NEXT: s_cbranch_execnz .LBB28_49832; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=19833; GFX942-NEXT: s_mov_b64 exec, s[8:9]9834; GFX942-NEXT: s_waitcnt vmcnt(0)9835; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v99836; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]9837; GFX942-NEXT: v_mov_b32_e32 v9, v69838; GFX942-NEXT: buffer_inv sc19839; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]9840; GFX942-NEXT: s_cbranch_execnz .LBB28_39841; GFX942-NEXT: ; %bb.6: ; %atomicrmw.end9842; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]9843; GFX942-NEXT: v_mov_b32_e32 v0, v69844; GFX942-NEXT: s_setpc_b64 s[30:31]9845;9846; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:9847; GFX11-TRUE16: ; %bb.0:9848; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9849; GFX11-TRUE16-NEXT: s_mov_b32 s1, 09850; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo9851; GFX11-TRUE16-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=19852; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v09853; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v19854; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v29855; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v39856; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)9857; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]9858; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]9859; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s09860; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9861; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s09862; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:10249863; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s09864; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_19865; GFX11-TRUE16-NEXT: ; %bb.2:9866; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s29867; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v59868; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v59869; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x19870; GFX11-TRUE16-NEXT: .p2align 69871; GFX11-TRUE16-NEXT: .LBB28_3: ; %atomicrmw.start9872; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=19873; GFX11-TRUE16-NEXT: ; Child Loop BB28_4 Depth 29874; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)9875; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v79876; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v79877; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo9878; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x09879; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9880; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, v6, v8 :: v_dual_add_f32 v5, v5, v99881; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 19882; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)9883; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 19884; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v59885; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v59886; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v69887; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff9888; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff9889; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)9890; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo9891; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v69892; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo9893; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)9894; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.h9895; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v69896; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v79897; GFX11-TRUE16-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=19898; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=29899; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v09900; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v19901; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v29902; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v39903; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)9904; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]9905; GFX11-TRUE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]9906; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s09907; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9908; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s09909; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)9910; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc9911; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s09912; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_49913; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=19914; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s29915; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)9916; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v79917; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v59918; GFX11-TRUE16-NEXT: buffer_gl1_inv9919; GFX11-TRUE16-NEXT: buffer_gl0_inv9920; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s19921; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9922; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s19923; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_39924; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end9925; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x29926; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s19927; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v59928; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]9929;9930; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:9931; GFX11-FAKE16: ; %bb.0:9932; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9933; GFX11-FAKE16-NEXT: s_mov_b32 s1, 09934; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo9935; GFX11-FAKE16-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=19936; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v09937; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v19938; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v29939; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v39940; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)9941; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]9942; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]9943; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s09944; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9945; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s09946; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v4, s[4:7], 0 offen offset:10249947; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s09948; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_19949; GFX11-FAKE16-NEXT: ; %bb.2:9950; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s29951; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v59952; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v59953; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x19954; GFX11-FAKE16-NEXT: .p2align 69955; GFX11-FAKE16-NEXT: .LBB28_3: ; %atomicrmw.start9956; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=19957; GFX11-FAKE16-NEXT: ; Child Loop BB28_4 Depth 29958; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)9959; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v79960; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v79961; GFX11-FAKE16-NEXT: s_mov_b32 s2, exec_lo9962; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x09963; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9964; GFX11-FAKE16-NEXT: v_dual_add_f32 v6, v6, v9 :: v_dual_add_f32 v5, v5, v89965; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 19966; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)9967; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 19968; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v59969; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v59970; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, 0x400000, v69971; GFX11-FAKE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff9972; GFX11-FAKE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff9973; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)9974; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo9975; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v69976; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo9977; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9978; GFX11-FAKE16-NEXT: v_perm_b32 v6, v6, v5, 0x70603029979; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v69980; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v79981; GFX11-FAKE16-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=19982; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=29983; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v09984; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s5, v19985; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s6, v29986; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s7, v39987; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)9988; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[0:1]9989; GFX11-FAKE16-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[2:3]9990; GFX11-FAKE16-NEXT: s_and_b32 s0, vcc_lo, s09991; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9992; GFX11-FAKE16-NEXT: s_and_saveexec_b32 s0, s09993; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)9994; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[4:7], 0 offen offset:1024 glc9995; GFX11-FAKE16-NEXT: s_xor_b32 exec_lo, exec_lo, s09996; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_49997; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=19998; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s29999; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)10000; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v710001; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v510002; GFX11-FAKE16-NEXT: buffer_gl1_inv10003; GFX11-FAKE16-NEXT: buffer_gl0_inv10004; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s110005; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10006; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s110007; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_310008; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end10009; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x210010; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s110011; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v510012; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]10013;10014; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:10015; GFX10: ; %bb.0:10016; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10017; GFX10-NEXT: s_mov_b32 s5, 010018; GFX10-NEXT: s_mov_b32 s6, exec_lo10019; GFX10-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=110020; GFX10-NEXT: v_readfirstlane_b32 s8, v010021; GFX10-NEXT: v_readfirstlane_b32 s9, v110022; GFX10-NEXT: v_readfirstlane_b32 s10, v210023; GFX10-NEXT: v_readfirstlane_b32 s11, v310024; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]10025; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]10026; GFX10-NEXT: s_and_b32 s4, vcc_lo, s410027; GFX10-NEXT: s_and_saveexec_b32 s4, s410028; GFX10-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:102410029; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)10030; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s410031; GFX10-NEXT: s_cbranch_execnz .LBB28_110032; GFX10-NEXT: ; %bb.2:10033; GFX10-NEXT: s_mov_b32 exec_lo, s610034; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v510035; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v510036; GFX10-NEXT: .LBB28_3: ; %atomicrmw.start10037; GFX10-NEXT: ; =>This Loop Header: Depth=110038; GFX10-NEXT: ; Child Loop BB28_4 Depth 210039; GFX10-NEXT: s_waitcnt vmcnt(0)10040; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v710041; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v710042; GFX10-NEXT: s_mov_b32 s6, exec_lo10043; GFX10-NEXT: s_waitcnt_vscnt null, 0x010044; GFX10-NEXT: v_add_f32_e32 v5, v5, v810045; GFX10-NEXT: v_add_f32_e32 v6, v6, v910046; GFX10-NEXT: v_bfe_u32 v10, v5, 16, 110047; GFX10-NEXT: v_bfe_u32 v11, v6, 16, 110048; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v510049; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v510050; GFX10-NEXT: v_or_b32_e32 v13, 0x400000, v610051; GFX10-NEXT: v_add3_u32 v10, v10, v5, 0x7fff10052; GFX10-NEXT: v_add3_u32 v11, v11, v6, 0x7fff10053; GFX10-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo10054; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v610055; GFX10-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo10056; GFX10-NEXT: v_perm_b32 v6, v6, v5, 0x706030210057; GFX10-NEXT: v_mov_b32_e32 v5, v610058; GFX10-NEXT: v_mov_b32_e32 v6, v710059; GFX10-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=110060; GFX10-NEXT: ; => This Inner Loop Header: Depth=210061; GFX10-NEXT: v_readfirstlane_b32 s8, v010062; GFX10-NEXT: v_readfirstlane_b32 s9, v110063; GFX10-NEXT: v_readfirstlane_b32 s10, v210064; GFX10-NEXT: v_readfirstlane_b32 s11, v310065; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[8:9], v[0:1]10066; GFX10-NEXT: v_cmp_eq_u64_e64 s4, s[10:11], v[2:3]10067; GFX10-NEXT: s_and_b32 s4, vcc_lo, s410068; GFX10-NEXT: s_and_saveexec_b32 s4, s410069; GFX10-NEXT: s_waitcnt vmcnt(0)10070; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc10071; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)10072; GFX10-NEXT: s_xor_b32 exec_lo, exec_lo, s410073; GFX10-NEXT: s_cbranch_execnz .LBB28_410074; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=110075; GFX10-NEXT: s_mov_b32 exec_lo, s610076; GFX10-NEXT: s_waitcnt vmcnt(0)10077; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v710078; GFX10-NEXT: v_mov_b32_e32 v7, v510079; GFX10-NEXT: buffer_gl1_inv10080; GFX10-NEXT: buffer_gl0_inv10081; GFX10-NEXT: s_or_b32 s5, vcc_lo, s510082; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)10083; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s510084; GFX10-NEXT: s_cbranch_execnz .LBB28_310085; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end10086; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s510087; GFX10-NEXT: v_mov_b32_e32 v0, v510088; GFX10-NEXT: s_setpc_b64 s[30:31]10089;10090; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:10091; GFX90A: ; %bb.0:10092; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10093; GFX90A-NEXT: s_mov_b64 s[6:7], exec10094; GFX90A-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=110095; GFX90A-NEXT: v_readfirstlane_b32 s8, v010096; GFX90A-NEXT: v_readfirstlane_b32 s9, v110097; GFX90A-NEXT: v_readfirstlane_b32 s10, v210098; GFX90A-NEXT: v_readfirstlane_b32 s11, v310099; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]10100; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]10101; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]10102; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]10103; GFX90A-NEXT: s_nop 010104; GFX90A-NEXT: buffer_load_dword v9, v4, s[8:11], 0 offen offset:102410105; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]10106; GFX90A-NEXT: s_cbranch_execnz .LBB28_110107; GFX90A-NEXT: ; %bb.2:10108; GFX90A-NEXT: s_mov_b64 exec, s[6:7]10109; GFX90A-NEXT: s_mov_b64 s[6:7], 010110; GFX90A-NEXT: v_lshlrev_b32_e32 v10, 16, v510111; GFX90A-NEXT: s_movk_i32 s14, 0x7fff10112; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v510113; GFX90A-NEXT: s_mov_b32 s15, 0x706030210114; GFX90A-NEXT: .LBB28_3: ; %atomicrmw.start10115; GFX90A-NEXT: ; =>This Loop Header: Depth=110116; GFX90A-NEXT: ; Child Loop BB28_4 Depth 210117; GFX90A-NEXT: s_waitcnt vmcnt(0)10118; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v910119; GFX90A-NEXT: v_add_f32_e32 v6, v6, v1010120; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 110121; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s1410122; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v610123; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v610124; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc10125; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v910126; GFX90A-NEXT: v_add_f32_e32 v7, v7, v510127; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 110128; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s1410129; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v710130; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v710131; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc10132; GFX90A-NEXT: v_perm_b32 v8, v7, v6, s1510133; GFX90A-NEXT: s_mov_b64 s[12:13], exec10134; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]10135; GFX90A-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=110136; GFX90A-NEXT: ; => This Inner Loop Header: Depth=210137; GFX90A-NEXT: v_readfirstlane_b32 s8, v010138; GFX90A-NEXT: v_readfirstlane_b32 s9, v110139; GFX90A-NEXT: v_readfirstlane_b32 s10, v210140; GFX90A-NEXT: v_readfirstlane_b32 s11, v310141; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]10142; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]10143; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]10144; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]10145; GFX90A-NEXT: s_waitcnt vmcnt(0)10146; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc10147; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]10148; GFX90A-NEXT: s_cbranch_execnz .LBB28_410149; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=110150; GFX90A-NEXT: s_mov_b64 exec, s[12:13]10151; GFX90A-NEXT: s_waitcnt vmcnt(0)10152; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v910153; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10154; GFX90A-NEXT: v_mov_b32_e32 v9, v610155; GFX90A-NEXT: buffer_wbinvl110156; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]10157; GFX90A-NEXT: s_cbranch_execnz .LBB28_310158; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end10159; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]10160; GFX90A-NEXT: v_mov_b32_e32 v0, v610161; GFX90A-NEXT: s_setpc_b64 s[30:31]10162;10163; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:10164; GFX908: ; %bb.0:10165; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10166; GFX908-NEXT: s_mov_b64 s[6:7], exec10167; GFX908-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=110168; GFX908-NEXT: v_readfirstlane_b32 s8, v010169; GFX908-NEXT: v_readfirstlane_b32 s9, v110170; GFX908-NEXT: v_readfirstlane_b32 s10, v210171; GFX908-NEXT: v_readfirstlane_b32 s11, v310172; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]10173; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]10174; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]10175; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]10176; GFX908-NEXT: s_nop 010177; GFX908-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:102410178; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]10179; GFX908-NEXT: s_cbranch_execnz .LBB28_110180; GFX908-NEXT: ; %bb.2:10181; GFX908-NEXT: s_mov_b64 exec, s[6:7]10182; GFX908-NEXT: s_mov_b64 s[6:7], 010183; GFX908-NEXT: v_lshlrev_b32_e32 v8, 16, v510184; GFX908-NEXT: s_movk_i32 s14, 0x7fff10185; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v510186; GFX908-NEXT: s_mov_b32 s15, 0x706030210187; GFX908-NEXT: .LBB28_3: ; %atomicrmw.start10188; GFX908-NEXT: ; =>This Loop Header: Depth=110189; GFX908-NEXT: ; Child Loop BB28_4 Depth 210190; GFX908-NEXT: s_waitcnt vmcnt(0)10191; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v710192; GFX908-NEXT: v_add_f32_e32 v5, v5, v810193; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 110194; GFX908-NEXT: v_add3_u32 v6, v6, v5, s1410195; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v510196; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v510197; GFX908-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc10198; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v710199; GFX908-NEXT: v_add_f32_e32 v6, v6, v910200; GFX908-NEXT: v_bfe_u32 v10, v6, 16, 110201; GFX908-NEXT: v_add3_u32 v10, v10, v6, s1410202; GFX908-NEXT: v_or_b32_e32 v11, 0x400000, v610203; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v610204; GFX908-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc10205; GFX908-NEXT: v_perm_b32 v6, v6, v5, s1510206; GFX908-NEXT: v_mov_b32_e32 v5, v610207; GFX908-NEXT: s_mov_b64 s[12:13], exec10208; GFX908-NEXT: v_mov_b32_e32 v6, v710209; GFX908-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=110210; GFX908-NEXT: ; => This Inner Loop Header: Depth=210211; GFX908-NEXT: v_readfirstlane_b32 s8, v010212; GFX908-NEXT: v_readfirstlane_b32 s9, v110213; GFX908-NEXT: v_readfirstlane_b32 s10, v210214; GFX908-NEXT: v_readfirstlane_b32 s11, v310215; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]10216; GFX908-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]10217; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]10218; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]10219; GFX908-NEXT: s_waitcnt vmcnt(0)10220; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc10221; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]10222; GFX908-NEXT: s_cbranch_execnz .LBB28_410223; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=110224; GFX908-NEXT: s_mov_b64 exec, s[12:13]10225; GFX908-NEXT: s_waitcnt vmcnt(0)10226; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v710227; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10228; GFX908-NEXT: v_mov_b32_e32 v7, v510229; GFX908-NEXT: buffer_wbinvl110230; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]10231; GFX908-NEXT: s_cbranch_execnz .LBB28_310232; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end10233; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]10234; GFX908-NEXT: v_mov_b32_e32 v0, v510235; GFX908-NEXT: s_setpc_b64 s[30:31]10236;10237; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:10238; GFX8: ; %bb.0:10239; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10240; GFX8-NEXT: s_mov_b64 s[6:7], exec10241; GFX8-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=110242; GFX8-NEXT: v_readfirstlane_b32 s8, v010243; GFX8-NEXT: v_readfirstlane_b32 s9, v110244; GFX8-NEXT: v_readfirstlane_b32 s10, v210245; GFX8-NEXT: v_readfirstlane_b32 s11, v310246; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]10247; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]10248; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]10249; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]10250; GFX8-NEXT: s_nop 010251; GFX8-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:102410252; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]10253; GFX8-NEXT: s_cbranch_execnz .LBB28_110254; GFX8-NEXT: ; %bb.2:10255; GFX8-NEXT: s_mov_b64 exec, s[6:7]10256; GFX8-NEXT: s_mov_b64 s[6:7], 010257; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v510258; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v510259; GFX8-NEXT: .LBB28_3: ; %atomicrmw.start10260; GFX8-NEXT: ; =>This Loop Header: Depth=110261; GFX8-NEXT: ; Child Loop BB28_4 Depth 210262; GFX8-NEXT: s_waitcnt vmcnt(0)10263; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v710264; GFX8-NEXT: v_add_f32_e32 v5, v5, v810265; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 110266; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v510267; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v610268; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v510269; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v510270; GFX8-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc10271; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v710272; GFX8-NEXT: v_add_f32_e32 v6, v6, v910273; GFX8-NEXT: v_bfe_u32 v10, v6, 16, 110274; GFX8-NEXT: v_add_u32_e32 v10, vcc, v10, v610275; GFX8-NEXT: v_add_u32_e32 v10, vcc, 0x7fff, v1010276; GFX8-NEXT: v_or_b32_e32 v11, 0x400000, v610277; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v610278; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc10279; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v610280; GFX8-NEXT: v_alignbit_b32 v6, v6, v5, 1610281; GFX8-NEXT: v_mov_b32_e32 v5, v610282; GFX8-NEXT: s_mov_b64 s[12:13], exec10283; GFX8-NEXT: v_mov_b32_e32 v6, v710284; GFX8-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=110285; GFX8-NEXT: ; => This Inner Loop Header: Depth=210286; GFX8-NEXT: v_readfirstlane_b32 s8, v010287; GFX8-NEXT: v_readfirstlane_b32 s9, v110288; GFX8-NEXT: v_readfirstlane_b32 s10, v210289; GFX8-NEXT: v_readfirstlane_b32 s11, v310290; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]10291; GFX8-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]10292; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]10293; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]10294; GFX8-NEXT: s_waitcnt vmcnt(0)10295; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc10296; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]10297; GFX8-NEXT: s_cbranch_execnz .LBB28_410298; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=110299; GFX8-NEXT: s_mov_b64 exec, s[12:13]10300; GFX8-NEXT: s_waitcnt vmcnt(0)10301; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v710302; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10303; GFX8-NEXT: v_mov_b32_e32 v7, v510304; GFX8-NEXT: buffer_wbinvl110305; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]10306; GFX8-NEXT: s_cbranch_execnz .LBB28_310307; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end10308; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]10309; GFX8-NEXT: v_mov_b32_e32 v0, v510310; GFX8-NEXT: s_setpc_b64 s[30:31]10311;10312; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:10313; GFX7: ; %bb.0:10314; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10315; GFX7-NEXT: s_mov_b64 s[6:7], exec10316; GFX7-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=110317; GFX7-NEXT: v_readfirstlane_b32 s8, v010318; GFX7-NEXT: v_readfirstlane_b32 s9, v110319; GFX7-NEXT: v_readfirstlane_b32 s10, v210320; GFX7-NEXT: v_readfirstlane_b32 s11, v310321; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]10322; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]10323; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]10324; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]10325; GFX7-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:102410326; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]10327; GFX7-NEXT: s_cbranch_execnz .LBB28_110328; GFX7-NEXT: ; %bb.2:10329; GFX7-NEXT: s_mov_b64 exec, s[6:7]10330; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v510331; GFX7-NEXT: v_and_b32_e32 v10, 0xffff0000, v510332; GFX7-NEXT: v_mul_f32_e32 v5, 1.0, v610333; GFX7-NEXT: s_waitcnt vmcnt(0)10334; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v810335; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v810336; GFX7-NEXT: s_mov_b64 s[6:7], 010337; GFX7-NEXT: v_and_b32_e32 v11, 0xffff0000, v510338; GFX7-NEXT: .LBB28_3: ; %atomicrmw.start10339; GFX7-NEXT: ; =>This Loop Header: Depth=110340; GFX7-NEXT: ; Child Loop BB28_4 Depth 210341; GFX7-NEXT: v_mul_f32_e32 v7, 1.0, v710342; GFX7-NEXT: v_mul_f32_e32 v6, 1.0, v810343; GFX7-NEXT: v_and_b32_e32 v8, 0xffff0000, v710344; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v610345; GFX7-NEXT: v_add_f32_e32 v8, v8, v1110346; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v710347; GFX7-NEXT: v_add_f32_e32 v5, v5, v1010348; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v810349; GFX7-NEXT: v_alignbit_b32 v6, v7, v6, 1610350; GFX7-NEXT: v_alignbit_b32 v5, v8, v5, 1610351; GFX7-NEXT: v_mov_b32_e32 v9, v610352; GFX7-NEXT: s_mov_b64 s[12:13], exec10353; GFX7-NEXT: v_mov_b32_e32 v8, v510354; GFX7-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=110355; GFX7-NEXT: ; => This Inner Loop Header: Depth=210356; GFX7-NEXT: v_readfirstlane_b32 s8, v010357; GFX7-NEXT: v_readfirstlane_b32 s9, v110358; GFX7-NEXT: v_readfirstlane_b32 s10, v210359; GFX7-NEXT: v_readfirstlane_b32 s11, v310360; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]10361; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]10362; GFX7-NEXT: s_and_b64 s[4:5], vcc, s[4:5]10363; GFX7-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]10364; GFX7-NEXT: s_waitcnt vmcnt(0)10365; GFX7-NEXT: buffer_atomic_cmpswap v[8:9], v4, s[8:11], 0 offen offset:1024 glc10366; GFX7-NEXT: s_xor_b64 exec, exec, s[4:5]10367; GFX7-NEXT: s_cbranch_execnz .LBB28_410368; GFX7-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=110369; GFX7-NEXT: s_mov_b64 exec, s[12:13]10370; GFX7-NEXT: s_waitcnt vmcnt(0)10371; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v8, v610372; GFX7-NEXT: v_and_b32_e32 v7, 0xffff0000, v810373; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10374; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v810375; GFX7-NEXT: buffer_wbinvl110376; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]10377; GFX7-NEXT: s_cbranch_execnz .LBB28_310378; GFX7-NEXT: ; %bb.6: ; %atomicrmw.end10379; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]10380; GFX7-NEXT: v_mov_b32_e32 v0, v810381; GFX7-NEXT: v_mov_b32_e32 v1, v710382; GFX7-NEXT: s_setpc_b64 s[30:31]10383;10384; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:10385; GFX6: ; %bb.0:10386; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10387; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x400, v410388; GFX6-NEXT: s_mov_b64 s[6:7], exec10389; GFX6-NEXT: .LBB28_1: ; =>This Inner Loop Header: Depth=110390; GFX6-NEXT: v_readfirstlane_b32 s8, v010391; GFX6-NEXT: v_readfirstlane_b32 s9, v110392; GFX6-NEXT: v_readfirstlane_b32 s10, v210393; GFX6-NEXT: v_readfirstlane_b32 s11, v310394; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]10395; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]10396; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]10397; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]10398; GFX6-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:102410399; GFX6-NEXT: ; implicit-def: $vgpr410400; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]10401; GFX6-NEXT: s_cbranch_execnz .LBB28_110402; GFX6-NEXT: ; %bb.2:10403; GFX6-NEXT: s_mov_b64 exec, s[6:7]10404; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v510405; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v510406; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v610407; GFX6-NEXT: s_waitcnt vmcnt(0)10408; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v710409; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v710410; GFX6-NEXT: s_mov_b64 s[6:7], 010411; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v510412; GFX6-NEXT: .LBB28_3: ; %atomicrmw.start10413; GFX6-NEXT: ; =>This Loop Header: Depth=110414; GFX6-NEXT: ; Child Loop BB28_4 Depth 210415; GFX6-NEXT: v_mul_f32_e32 v5, 1.0, v710416; GFX6-NEXT: v_mul_f32_e32 v7, 1.0, v410417; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v710418; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v510419; GFX6-NEXT: v_add_f32_e32 v4, v4, v1010420; GFX6-NEXT: v_add_f32_e32 v6, v6, v910421; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v410422; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 1610423; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v710424; GFX6-NEXT: v_alignbit_b32 v5, v6, v5, 1610425; GFX6-NEXT: v_mov_b32_e32 v7, v510426; GFX6-NEXT: s_mov_b64 s[12:13], exec10427; GFX6-NEXT: v_mov_b32_e32 v6, v410428; GFX6-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=110429; GFX6-NEXT: ; => This Inner Loop Header: Depth=210430; GFX6-NEXT: v_readfirstlane_b32 s8, v010431; GFX6-NEXT: v_readfirstlane_b32 s9, v110432; GFX6-NEXT: v_readfirstlane_b32 s10, v210433; GFX6-NEXT: v_readfirstlane_b32 s11, v310434; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]10435; GFX6-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[2:3]10436; GFX6-NEXT: s_and_b64 s[4:5], vcc, s[4:5]10437; GFX6-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]10438; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)10439; GFX6-NEXT: buffer_atomic_cmpswap v[6:7], v8, s[8:11], 0 offen glc10440; GFX6-NEXT: s_xor_b64 exec, exec, s[4:5]10441; GFX6-NEXT: s_cbranch_execnz .LBB28_410442; GFX6-NEXT: ; %bb.5: ; in Loop: Header=BB28_3 Depth=110443; GFX6-NEXT: s_mov_b64 exec, s[12:13]10444; GFX6-NEXT: s_waitcnt vmcnt(0)10445; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v6, v510446; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v610447; GFX6-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10448; GFX6-NEXT: s_waitcnt expcnt(0)10449; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v610450; GFX6-NEXT: buffer_wbinvl110451; GFX6-NEXT: s_andn2_b64 exec, exec, s[6:7]10452; GFX6-NEXT: s_cbranch_execnz .LBB28_310453; GFX6-NEXT: ; %bb.6: ; %atomicrmw.end10454; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]10455; GFX6-NEXT: v_mov_b32_e32 v0, v710456; GFX6-NEXT: v_mov_b32_e32 v1, v410457; GFX6-NEXT: s_setpc_b64 s[30:31]10458 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 25610459 %result = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !010460 ret <2 x bfloat> %result10461}10462 10463define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) {10464; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:10465; GFX12: ; %bb.0:10466; GFX12-NEXT: s_wait_loadcnt_dscnt 0x010467; GFX12-NEXT: s_wait_expcnt 0x010468; GFX12-NEXT: s_wait_samplecnt 0x010469; GFX12-NEXT: s_wait_bvhcnt 0x010470; GFX12-NEXT: s_wait_kmcnt 0x010471; GFX12-NEXT: v_mov_b32_e32 v1, s1610472; GFX12-NEXT: s_wait_storecnt 0x010473; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN10474; GFX12-NEXT: s_wait_loadcnt 0x010475; GFX12-NEXT: global_inv scope:SCOPE_DEV10476; GFX12-NEXT: s_wait_loadcnt 0x010477; GFX12-NEXT: s_setpc_b64 s[30:31]10478;10479; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:10480; GFX942: ; %bb.0:10481; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10482; GFX942-NEXT: v_mov_b32_e32 v1, v010483; GFX942-NEXT: v_mov_b32_e32 v0, s1610484; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:102410485; GFX942-NEXT: s_mov_b64 s[6:7], 010486; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v110487; GFX942-NEXT: s_movk_i32 s8, 0x7fff10488; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v110489; GFX942-NEXT: s_mov_b32 s9, 0x706030210490; GFX942-NEXT: v_mov_b32_e32 v4, s1610491; GFX942-NEXT: .LBB29_1: ; %atomicrmw.start10492; GFX942-NEXT: ; =>This Inner Loop Header: Depth=110493; GFX942-NEXT: s_waitcnt vmcnt(0)10494; GFX942-NEXT: v_mov_b32_e32 v7, v010495; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v710496; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v710497; GFX942-NEXT: v_add_f32_e32 v0, v0, v210498; GFX942-NEXT: v_add_f32_e32 v1, v1, v310499; GFX942-NEXT: v_bfe_u32 v5, v0, 16, 110500; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 110501; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v010502; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v110503; GFX942-NEXT: v_add3_u32 v5, v5, v0, s810504; GFX942-NEXT: v_add3_u32 v8, v8, v1, s810505; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v110506; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v010507; GFX942-NEXT: buffer_wbl2 sc110508; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc10509; GFX942-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]10510; GFX942-NEXT: v_perm_b32 v6, v1, v0, s910511; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[6:7]10512; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[0:3], 0 offen offset:1024 sc010513; GFX942-NEXT: s_waitcnt vmcnt(0)10514; GFX942-NEXT: buffer_inv sc110515; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v710516; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10517; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]10518; GFX942-NEXT: s_cbranch_execnz .LBB29_110519; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end10520; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]10521; GFX942-NEXT: s_setpc_b64 s[30:31]10522;10523; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:10524; GFX11-TRUE16: ; %bb.0:10525; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10526; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s1610527; GFX11-TRUE16-NEXT: s_mov_b32 s4, 010528; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10529; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v110530; GFX11-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:102410531; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v110532; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x110533; GFX11-TRUE16-NEXT: .p2align 610534; GFX11-TRUE16-NEXT: .LBB29_1: ; %atomicrmw.start10535; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=110536; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)10537; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v010538; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x010539; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10540; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v610541; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v210542; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)10543; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 110544; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v110545; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff10546; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v610547; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10548; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v310549; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 110550; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v010551; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v010552; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)10553; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff10554; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo10555; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v110556; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo10557; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)10558; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h10559; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v610560; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc10561; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)10562; GFX11-TRUE16-NEXT: buffer_gl1_inv10563; GFX11-TRUE16-NEXT: buffer_gl0_inv10564; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v610565; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s410566; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10567; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s410568; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB29_110569; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end10570; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x210571; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s410572; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]10573;10574; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:10575; GFX11-FAKE16: ; %bb.0:10576; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10577; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s1610578; GFX11-FAKE16-NEXT: s_mov_b32 s5, 010579; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10580; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v110581; GFX11-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:102410582; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v110583; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x110584; GFX11-FAKE16-NEXT: .p2align 610585; GFX11-FAKE16-NEXT: .LBB29_1: ; %atomicrmw.start10586; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=110587; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)10588; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v010589; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x010590; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10591; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v610592; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v310593; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)10594; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 110595; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v110596; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v110597; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff10598; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10599; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v610600; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v210601; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)10602; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 110603; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v010604; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v010605; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff10606; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10607; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s410608; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x706030210609; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)10610; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v610611; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc10612; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)10613; GFX11-FAKE16-NEXT: buffer_gl1_inv10614; GFX11-FAKE16-NEXT: buffer_gl0_inv10615; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v610616; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s510617; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10618; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s510619; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB29_110620; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end10621; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x210622; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s510623; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]10624;10625; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:10626; GFX10: ; %bb.0:10627; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10628; GFX10-NEXT: v_mov_b32_e32 v1, v010629; GFX10-NEXT: v_mov_b32_e32 v0, s2010630; GFX10-NEXT: v_mov_b32_e32 v4, s2010631; GFX10-NEXT: s_mov_b32 s5, 010632; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v110633; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102410634; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v110635; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start10636; GFX10-NEXT: ; =>This Inner Loop Header: Depth=110637; GFX10-NEXT: s_waitcnt vmcnt(0)10638; GFX10-NEXT: v_mov_b32_e32 v6, v010639; GFX10-NEXT: s_waitcnt_vscnt null, 0x010640; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v610641; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v610642; GFX10-NEXT: v_add_f32_e32 v0, v0, v210643; GFX10-NEXT: v_add_f32_e32 v1, v1, v310644; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 110645; GFX10-NEXT: v_bfe_u32 v7, v1, 16, 110646; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v010647; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v110648; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v110649; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff10650; GFX10-NEXT: v_add3_u32 v7, v7, v1, 0x7fff10651; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v010652; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo10653; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v8, s410654; GFX10-NEXT: v_perm_b32 v5, v1, v0, 0x706030210655; GFX10-NEXT: v_mov_b32_e32 v0, v510656; GFX10-NEXT: v_mov_b32_e32 v1, v610657; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc10658; GFX10-NEXT: s_waitcnt vmcnt(0)10659; GFX10-NEXT: buffer_gl1_inv10660; GFX10-NEXT: buffer_gl0_inv10661; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v610662; GFX10-NEXT: s_or_b32 s5, vcc_lo, s510663; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s510664; GFX10-NEXT: s_cbranch_execnz .LBB29_110665; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end10666; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s510667; GFX10-NEXT: s_setpc_b64 s[30:31]10668;10669; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:10670; GFX90A: ; %bb.0:10671; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10672; GFX90A-NEXT: v_mov_b32_e32 v1, v010673; GFX90A-NEXT: v_mov_b32_e32 v0, s2010674; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102410675; GFX90A-NEXT: s_mov_b64 s[6:7], 010676; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v110677; GFX90A-NEXT: s_movk_i32 s8, 0x7fff10678; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v110679; GFX90A-NEXT: s_mov_b32 s9, 0x706030210680; GFX90A-NEXT: v_mov_b32_e32 v4, s2010681; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start10682; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=110683; GFX90A-NEXT: s_waitcnt vmcnt(0)10684; GFX90A-NEXT: v_mov_b32_e32 v7, v010685; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v710686; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v710687; GFX90A-NEXT: v_add_f32_e32 v0, v0, v210688; GFX90A-NEXT: v_add_f32_e32 v1, v1, v310689; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 110690; GFX90A-NEXT: v_bfe_u32 v8, v1, 16, 110691; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v010692; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v110693; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s810694; GFX90A-NEXT: v_add3_u32 v8, v8, v1, s810695; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v110696; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v010697; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]10698; GFX90A-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc10699; GFX90A-NEXT: v_perm_b32 v6, v1, v0, s910700; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]10701; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc10702; GFX90A-NEXT: s_waitcnt vmcnt(0)10703; GFX90A-NEXT: buffer_wbinvl110704; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v710705; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10706; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]10707; GFX90A-NEXT: s_cbranch_execnz .LBB29_110708; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end10709; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]10710; GFX90A-NEXT: s_setpc_b64 s[30:31]10711;10712; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:10713; GFX908: ; %bb.0:10714; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10715; GFX908-NEXT: v_mov_b32_e32 v1, v010716; GFX908-NEXT: v_mov_b32_e32 v0, s2010717; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102410718; GFX908-NEXT: s_mov_b64 s[6:7], 010719; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v110720; GFX908-NEXT: s_movk_i32 s8, 0x7fff10721; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v110722; GFX908-NEXT: s_mov_b32 s9, 0x706030210723; GFX908-NEXT: v_mov_b32_e32 v4, s2010724; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start10725; GFX908-NEXT: ; =>This Inner Loop Header: Depth=110726; GFX908-NEXT: s_waitcnt vmcnt(0)10727; GFX908-NEXT: v_mov_b32_e32 v6, v010728; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v610729; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v610730; GFX908-NEXT: v_add_f32_e32 v0, v0, v210731; GFX908-NEXT: v_add_f32_e32 v1, v1, v310732; GFX908-NEXT: v_bfe_u32 v5, v0, 16, 110733; GFX908-NEXT: v_bfe_u32 v8, v1, 16, 110734; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v010735; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v110736; GFX908-NEXT: v_add3_u32 v5, v5, v0, s810737; GFX908-NEXT: v_add3_u32 v8, v8, v1, s810738; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v110739; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v010740; GFX908-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]10741; GFX908-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc10742; GFX908-NEXT: v_perm_b32 v5, v1, v0, s910743; GFX908-NEXT: v_mov_b32_e32 v0, v510744; GFX908-NEXT: v_mov_b32_e32 v1, v610745; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc10746; GFX908-NEXT: s_waitcnt vmcnt(0)10747; GFX908-NEXT: buffer_wbinvl110748; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v610749; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10750; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]10751; GFX908-NEXT: s_cbranch_execnz .LBB29_110752; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end10753; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]10754; GFX908-NEXT: s_setpc_b64 s[30:31]10755;10756; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:10757; GFX8: ; %bb.0:10758; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10759; GFX8-NEXT: v_mov_b32_e32 v1, v010760; GFX8-NEXT: v_mov_b32_e32 v0, s2010761; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102410762; GFX8-NEXT: s_mov_b64 s[6:7], 010763; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v110764; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v110765; GFX8-NEXT: v_mov_b32_e32 v4, s2010766; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start10767; GFX8-NEXT: ; =>This Inner Loop Header: Depth=110768; GFX8-NEXT: s_waitcnt vmcnt(0)10769; GFX8-NEXT: v_mov_b32_e32 v6, v010770; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v610771; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v610772; GFX8-NEXT: v_add_f32_e32 v0, v0, v210773; GFX8-NEXT: v_add_f32_e32 v1, v1, v310774; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 110775; GFX8-NEXT: v_bfe_u32 v8, v1, 16, 110776; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v010777; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v110778; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v510779; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v810780; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v110781; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v110782; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v010783; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v010784; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc10785; GFX8-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]10786; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v110787; GFX8-NEXT: v_alignbit_b32 v5, v1, v0, 1610788; GFX8-NEXT: v_mov_b32_e32 v0, v510789; GFX8-NEXT: v_mov_b32_e32 v1, v610790; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc10791; GFX8-NEXT: s_waitcnt vmcnt(0)10792; GFX8-NEXT: buffer_wbinvl110793; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v610794; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10795; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]10796; GFX8-NEXT: s_cbranch_execnz .LBB29_110797; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end10798; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]10799; GFX8-NEXT: s_setpc_b64 s[30:31]10800;10801; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:10802; GFX7: ; %bb.0:10803; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10804; GFX7-NEXT: v_mov_b32_e32 v2, s2010805; GFX7-NEXT: buffer_load_dword v4, v2, s[16:19], 0 offen offset:102410806; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v010807; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v110808; GFX7-NEXT: s_mov_b64 s[4:5], 010809; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v010810; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v110811; GFX7-NEXT: s_waitcnt vmcnt(0)10812; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v410813; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v410814; GFX7-NEXT: v_mov_b32_e32 v4, s2010815; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start10816; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110817; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v110818; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v010819; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v110820; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v010821; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v110822; GFX7-NEXT: v_add_f32_e32 v6, v6, v310823; GFX7-NEXT: v_add_f32_e32 v5, v5, v210824; GFX7-NEXT: v_alignbit_b32 v1, v1, v0, 1610825; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v610826; GFX7-NEXT: v_alignbit_b32 v0, v0, v5, 1610827; GFX7-NEXT: v_mov_b32_e32 v6, v110828; GFX7-NEXT: v_mov_b32_e32 v5, v010829; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc10830; GFX7-NEXT: s_waitcnt vmcnt(0)10831; GFX7-NEXT: buffer_wbinvl110832; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v110833; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v510834; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10835; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v510836; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]10837; GFX7-NEXT: s_cbranch_execnz .LBB29_110838; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10839; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]10840; GFX7-NEXT: s_setpc_b64 s[30:31]10841;10842; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:10843; GFX6: ; %bb.0:10844; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10845; GFX6-NEXT: v_mov_b32_e32 v2, s2010846; GFX6-NEXT: buffer_load_dword v4, v2, s[16:19], 0 offen offset:102410847; GFX6-NEXT: s_add_i32 s6, s20, 0x40010848; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v010849; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v110850; GFX6-NEXT: s_mov_b64 s[4:5], 010851; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v010852; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v110853; GFX6-NEXT: s_waitcnt vmcnt(0)10854; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v410855; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v410856; GFX6-NEXT: v_mov_b32_e32 v4, s610857; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start10858; GFX6-NEXT: ; =>This Inner Loop Header: Depth=110859; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v110860; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v010861; GFX6-NEXT: s_waitcnt expcnt(0)10862; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v110863; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v010864; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v110865; GFX6-NEXT: v_add_f32_e32 v6, v6, v310866; GFX6-NEXT: v_add_f32_e32 v5, v5, v210867; GFX6-NEXT: v_alignbit_b32 v1, v1, v0, 1610868; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v610869; GFX6-NEXT: v_alignbit_b32 v0, v0, v5, 1610870; GFX6-NEXT: v_mov_b32_e32 v6, v110871; GFX6-NEXT: v_mov_b32_e32 v5, v010872; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen glc10873; GFX6-NEXT: s_waitcnt vmcnt(0)10874; GFX6-NEXT: buffer_wbinvl110875; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v110876; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v510877; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10878; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v510879; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]10880; GFX6-NEXT: s_cbranch_execnz .LBB29_110881; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end10882; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]10883; GFX6-NEXT: s_waitcnt expcnt(0)10884; GFX6-NEXT: s_setpc_b64 s[30:31]10885 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 25610886 %result = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst10887 ret <2 x bfloat> %result10888}10889 10890define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) {10891; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:10892; GFX12: ; %bb.0:10893; GFX12-NEXT: s_wait_loadcnt_dscnt 0x010894; GFX12-NEXT: s_wait_expcnt 0x010895; GFX12-NEXT: s_wait_samplecnt 0x010896; GFX12-NEXT: s_wait_bvhcnt 0x010897; GFX12-NEXT: s_wait_kmcnt 0x010898; GFX12-NEXT: v_mov_b32_e32 v1, s1610899; GFX12-NEXT: s_wait_storecnt 0x010900; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:102410901; GFX12-NEXT: s_wait_storecnt 0x010902; GFX12-NEXT: global_inv scope:SCOPE_DEV10903; GFX12-NEXT: s_wait_loadcnt 0x010904; GFX12-NEXT: s_setpc_b64 s[30:31]10905;10906; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:10907; GFX942: ; %bb.0:10908; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10909; GFX942-NEXT: v_mov_b32_e32 v1, s1610910; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:102410911; GFX942-NEXT: s_mov_b64 s[6:7], 010912; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v010913; GFX942-NEXT: s_movk_i32 s8, 0x7fff10914; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v010915; GFX942-NEXT: s_mov_b32 s9, 0x706030210916; GFX942-NEXT: v_mov_b32_e32 v4, s1610917; GFX942-NEXT: .LBB30_1: ; %atomicrmw.start10918; GFX942-NEXT: ; =>This Inner Loop Header: Depth=110919; GFX942-NEXT: s_waitcnt vmcnt(0)10920; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v110921; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v110922; GFX942-NEXT: v_add_f32_e32 v0, v0, v210923; GFX942-NEXT: v_add_f32_e32 v5, v5, v310924; GFX942-NEXT: v_bfe_u32 v6, v0, 16, 110925; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 110926; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v010927; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v510928; GFX942-NEXT: v_add3_u32 v6, v6, v0, s810929; GFX942-NEXT: v_add3_u32 v8, v8, v5, s810930; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v510931; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v010932; GFX942-NEXT: buffer_wbl2 sc110933; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc10934; GFX942-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]10935; GFX942-NEXT: v_perm_b32 v0, v5, v0, s910936; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1]10937; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen offset:1024 sc010938; GFX942-NEXT: s_waitcnt vmcnt(0)10939; GFX942-NEXT: buffer_inv sc110940; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v110941; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]10942; GFX942-NEXT: v_mov_b32_e32 v1, v610943; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]10944; GFX942-NEXT: s_cbranch_execnz .LBB30_110945; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end10946; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]10947; GFX942-NEXT: s_setpc_b64 s[30:31]10948;10949; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:10950; GFX11-TRUE16: ; %bb.0:10951; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10952; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v010953; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v010954; GFX11-TRUE16-NEXT: s_mov_b32 s4, 010955; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:102410956; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x110957; GFX11-TRUE16-NEXT: .p2align 610958; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start10959; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=110960; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)10961; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v110962; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v110963; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x010964; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)10965; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v2 :: v_dual_add_f32 v0, v0, v310966; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 110967; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)10968; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 110969; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v010970; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v010971; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v510972; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff10973; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff10974; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)10975; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo10976; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v510977; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo10978; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)10979; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h10980; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v010981; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc10982; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)10983; GFX11-TRUE16-NEXT: buffer_gl1_inv10984; GFX11-TRUE16-NEXT: buffer_gl0_inv10985; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v110986; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v510987; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s410988; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)10989; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s410990; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB30_110991; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end10992; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x210993; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s410994; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]10995;10996; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:10997; GFX11-FAKE16: ; %bb.0:10998; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10999; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v011000; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v011001; GFX11-FAKE16-NEXT: s_mov_b32 s5, 011002; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:102411003; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x111004; GFX11-FAKE16-NEXT: .p2align 611005; GFX11-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start11006; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=111007; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)11008; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v111009; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v111010; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x011011; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11012; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v211013; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 111014; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)11015; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 111016; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v011017; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v511018; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v511019; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff11020; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff11021; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v011022; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)11023; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo11024; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s411025; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11026; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x706030211027; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v011028; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc11029; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)11030; GFX11-FAKE16-NEXT: buffer_gl1_inv11031; GFX11-FAKE16-NEXT: buffer_gl0_inv11032; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v111033; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v511034; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s511035; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11036; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s511037; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB30_111038; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end11039; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x211040; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s511041; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]11042;11043; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:11044; GFX10: ; %bb.0:11045; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11046; GFX10-NEXT: v_mov_b32_e32 v1, s2011047; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v011048; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v011049; GFX10-NEXT: v_mov_b32_e32 v4, s2011050; GFX10-NEXT: s_mov_b32 s5, 011051; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102411052; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start11053; GFX10-NEXT: ; =>This Inner Loop Header: Depth=111054; GFX10-NEXT: s_waitcnt vmcnt(0)11055; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v111056; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v111057; GFX10-NEXT: s_waitcnt_vscnt null, 0x011058; GFX10-NEXT: v_add_f32_e32 v0, v0, v211059; GFX10-NEXT: v_add_f32_e32 v5, v5, v311060; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 111061; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 111062; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v011063; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v511064; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v511065; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff11066; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff11067; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v011068; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo11069; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s411070; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x706030211071; GFX10-NEXT: v_mov_b32_e32 v6, v111072; GFX10-NEXT: v_mov_b32_e32 v5, v011073; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc11074; GFX10-NEXT: s_waitcnt vmcnt(0)11075; GFX10-NEXT: buffer_gl1_inv11076; GFX10-NEXT: buffer_gl0_inv11077; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v111078; GFX10-NEXT: v_mov_b32_e32 v1, v511079; GFX10-NEXT: s_or_b32 s5, vcc_lo, s511080; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s511081; GFX10-NEXT: s_cbranch_execnz .LBB30_111082; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end11083; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s511084; GFX10-NEXT: s_setpc_b64 s[30:31]11085;11086; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:11087; GFX90A: ; %bb.0:11088; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11089; GFX90A-NEXT: v_mov_b32_e32 v1, s2011090; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102411091; GFX90A-NEXT: s_mov_b64 s[6:7], 011092; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v011093; GFX90A-NEXT: s_movk_i32 s8, 0x7fff11094; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v011095; GFX90A-NEXT: s_mov_b32 s9, 0x706030211096; GFX90A-NEXT: v_mov_b32_e32 v4, s2011097; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start11098; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111099; GFX90A-NEXT: s_waitcnt vmcnt(0)11100; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v111101; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v111102; GFX90A-NEXT: v_add_f32_e32 v0, v0, v211103; GFX90A-NEXT: v_add_f32_e32 v5, v5, v311104; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 111105; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 111106; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v011107; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v511108; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s811109; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s811110; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v511111; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v011112; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]11113; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc11114; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s911115; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]11116; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen offset:1024 glc11117; GFX90A-NEXT: s_waitcnt vmcnt(0)11118; GFX90A-NEXT: buffer_wbinvl111119; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v111120; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11121; GFX90A-NEXT: v_mov_b32_e32 v1, v611122; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]11123; GFX90A-NEXT: s_cbranch_execnz .LBB30_111124; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11125; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]11126; GFX90A-NEXT: s_setpc_b64 s[30:31]11127;11128; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:11129; GFX908: ; %bb.0:11130; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11131; GFX908-NEXT: v_mov_b32_e32 v1, s2011132; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102411133; GFX908-NEXT: s_mov_b64 s[6:7], 011134; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v011135; GFX908-NEXT: s_movk_i32 s8, 0x7fff11136; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v011137; GFX908-NEXT: s_mov_b32 s9, 0x706030211138; GFX908-NEXT: v_mov_b32_e32 v4, s2011139; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start11140; GFX908-NEXT: ; =>This Inner Loop Header: Depth=111141; GFX908-NEXT: s_waitcnt vmcnt(0)11142; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v111143; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v111144; GFX908-NEXT: v_add_f32_e32 v0, v0, v211145; GFX908-NEXT: v_add_f32_e32 v5, v5, v311146; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 111147; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 111148; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v011149; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v511150; GFX908-NEXT: v_add3_u32 v6, v6, v0, s811151; GFX908-NEXT: v_add3_u32 v8, v8, v5, s811152; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v511153; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v011154; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]11155; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc11156; GFX908-NEXT: v_perm_b32 v0, v5, v0, s911157; GFX908-NEXT: v_mov_b32_e32 v6, v111158; GFX908-NEXT: v_mov_b32_e32 v5, v011159; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc11160; GFX908-NEXT: s_waitcnt vmcnt(0)11161; GFX908-NEXT: buffer_wbinvl111162; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v111163; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11164; GFX908-NEXT: v_mov_b32_e32 v1, v511165; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]11166; GFX908-NEXT: s_cbranch_execnz .LBB30_111167; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end11168; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]11169; GFX908-NEXT: s_setpc_b64 s[30:31]11170;11171; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:11172; GFX8: ; %bb.0:11173; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11174; GFX8-NEXT: v_mov_b32_e32 v1, s2011175; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102411176; GFX8-NEXT: s_mov_b64 s[6:7], 011177; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v011178; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v011179; GFX8-NEXT: v_mov_b32_e32 v4, s2011180; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start11181; GFX8-NEXT: ; =>This Inner Loop Header: Depth=111182; GFX8-NEXT: s_waitcnt vmcnt(0)11183; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v111184; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v111185; GFX8-NEXT: v_add_f32_e32 v0, v0, v211186; GFX8-NEXT: v_add_f32_e32 v5, v5, v311187; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 111188; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 111189; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v011190; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v511191; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v611192; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v811193; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v511194; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v511195; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v011196; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v011197; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc11198; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]11199; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v511200; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 1611201; GFX8-NEXT: v_mov_b32_e32 v6, v111202; GFX8-NEXT: v_mov_b32_e32 v5, v011203; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc11204; GFX8-NEXT: s_waitcnt vmcnt(0)11205; GFX8-NEXT: buffer_wbinvl111206; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v111207; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11208; GFX8-NEXT: v_mov_b32_e32 v1, v511209; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]11210; GFX8-NEXT: s_cbranch_execnz .LBB30_111211; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end11212; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]11213; GFX8-NEXT: s_setpc_b64 s[30:31]11214;11215; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:11216; GFX7: ; %bb.0:11217; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11218; GFX7-NEXT: v_mov_b32_e32 v2, s2011219; GFX7-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:102411220; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v011221; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v111222; GFX7-NEXT: s_mov_b64 s[4:5], 011223; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v011224; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v111225; GFX7-NEXT: s_waitcnt vmcnt(0)11226; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v211227; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v211228; GFX7-NEXT: v_mov_b32_e32 v2, s2011229; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start11230; GFX7-NEXT: ; =>This Inner Loop Header: Depth=111231; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v311232; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v411233; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v311234; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v411235; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v311236; GFX7-NEXT: v_add_f32_e32 v6, v6, v111237; GFX7-NEXT: v_add_f32_e32 v5, v5, v011238; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 1611239; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v611240; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 1611241; GFX7-NEXT: v_mov_b32_e32 v6, v411242; GFX7-NEXT: v_mov_b32_e32 v5, v311243; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc11244; GFX7-NEXT: s_waitcnt vmcnt(0)11245; GFX7-NEXT: buffer_wbinvl111246; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v411247; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v511248; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11249; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v511250; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]11251; GFX7-NEXT: s_cbranch_execnz .LBB30_111252; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end11253; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]11254; GFX7-NEXT: s_setpc_b64 s[30:31]11255;11256; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:11257; GFX6: ; %bb.0:11258; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11259; GFX6-NEXT: v_mov_b32_e32 v2, s2011260; GFX6-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:102411261; GFX6-NEXT: s_add_i32 s6, s20, 0x40011262; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v011263; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v111264; GFX6-NEXT: s_mov_b64 s[4:5], 011265; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v011266; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v111267; GFX6-NEXT: s_waitcnt vmcnt(0)11268; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v211269; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v211270; GFX6-NEXT: v_mov_b32_e32 v2, s611271; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start11272; GFX6-NEXT: ; =>This Inner Loop Header: Depth=111273; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v311274; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v411275; GFX6-NEXT: s_waitcnt expcnt(0)11276; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v311277; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v411278; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v311279; GFX6-NEXT: v_add_f32_e32 v6, v6, v111280; GFX6-NEXT: v_add_f32_e32 v5, v5, v011281; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 1611282; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v611283; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 1611284; GFX6-NEXT: v_mov_b32_e32 v6, v411285; GFX6-NEXT: v_mov_b32_e32 v5, v311286; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc11287; GFX6-NEXT: s_waitcnt vmcnt(0)11288; GFX6-NEXT: buffer_wbinvl111289; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v411290; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v511291; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11292; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v511293; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]11294; GFX6-NEXT: s_cbranch_execnz .LBB30_111295; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end11296; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]11297; GFX6-NEXT: s_waitcnt expcnt(0)11298; GFX6-NEXT: s_setpc_b64 s[30:31]11299 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 25611300 %unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst11301 ret void11302}11303 11304define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {11305; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:11306; GFX12: ; %bb.0:11307; GFX12-NEXT: s_wait_loadcnt_dscnt 0x011308; GFX12-NEXT: s_wait_expcnt 0x011309; GFX12-NEXT: s_wait_samplecnt 0x011310; GFX12-NEXT: s_wait_bvhcnt 0x011311; GFX12-NEXT: s_wait_kmcnt 0x011312; GFX12-NEXT: v_mov_b32_e32 v1, s1611313; GFX12-NEXT: s_wait_storecnt 0x011314; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN11315; GFX12-NEXT: s_wait_loadcnt 0x011316; GFX12-NEXT: global_inv scope:SCOPE_DEV11317; GFX12-NEXT: s_wait_loadcnt 0x011318; GFX12-NEXT: s_setpc_b64 s[30:31]11319;11320; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:11321; GFX942: ; %bb.0:11322; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11323; GFX942-NEXT: v_mov_b32_e32 v1, v011324; GFX942-NEXT: v_mov_b32_e32 v0, s1611325; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:102411326; GFX942-NEXT: s_mov_b64 s[6:7], 011327; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v111328; GFX942-NEXT: s_movk_i32 s8, 0x7fff11329; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v111330; GFX942-NEXT: s_mov_b32 s9, 0x706030211331; GFX942-NEXT: v_mov_b32_e32 v4, s1611332; GFX942-NEXT: .LBB31_1: ; %atomicrmw.start11333; GFX942-NEXT: ; =>This Inner Loop Header: Depth=111334; GFX942-NEXT: s_waitcnt vmcnt(0)11335; GFX942-NEXT: v_mov_b32_e32 v7, v011336; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v711337; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v711338; GFX942-NEXT: v_add_f32_e32 v0, v0, v211339; GFX942-NEXT: v_add_f32_e32 v1, v1, v311340; GFX942-NEXT: v_bfe_u32 v5, v0, 16, 111341; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 111342; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v011343; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v111344; GFX942-NEXT: v_add3_u32 v5, v5, v0, s811345; GFX942-NEXT: v_add3_u32 v8, v8, v1, s811346; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v111347; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v011348; GFX942-NEXT: buffer_wbl2 sc111349; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc11350; GFX942-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]11351; GFX942-NEXT: v_perm_b32 v6, v1, v0, s911352; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[6:7]11353; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[0:3], 0 offen offset:1024 sc011354; GFX942-NEXT: s_waitcnt vmcnt(0)11355; GFX942-NEXT: buffer_inv sc111356; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v711357; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11358; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]11359; GFX942-NEXT: s_cbranch_execnz .LBB31_111360; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end11361; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]11362; GFX942-NEXT: s_setpc_b64 s[30:31]11363;11364; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:11365; GFX11-TRUE16: ; %bb.0:11366; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11367; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s1611368; GFX11-TRUE16-NEXT: s_mov_b32 s4, 011369; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11370; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v111371; GFX11-TRUE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:102411372; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v111373; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x111374; GFX11-TRUE16-NEXT: .p2align 611375; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start11376; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=111377; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)11378; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v011379; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x011380; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11381; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v611382; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v211383; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)11384; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 111385; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v111386; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff11387; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v611388; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11389; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v311390; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 111391; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v011392; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v011393; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)11394; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff11395; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo11396; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v111397; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo11398; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)11399; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h11400; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v611401; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc11402; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)11403; GFX11-TRUE16-NEXT: buffer_gl1_inv11404; GFX11-TRUE16-NEXT: buffer_gl0_inv11405; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v611406; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s411407; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11408; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s411409; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB31_111410; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end11411; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x211412; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s411413; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]11414;11415; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:11416; GFX11-FAKE16: ; %bb.0:11417; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11418; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s1611419; GFX11-FAKE16-NEXT: s_mov_b32 s5, 011420; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11421; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v111422; GFX11-FAKE16-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:102411423; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v111424; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x111425; GFX11-FAKE16-NEXT: .p2align 611426; GFX11-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start11427; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=111428; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)11429; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v011430; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x011431; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11432; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v611433; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v311434; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)11435; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 111436; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v111437; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v111438; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff11439; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11440; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v611441; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v211442; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)11443; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 111444; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v011445; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v011446; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff11447; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11448; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s411449; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x706030211450; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)11451; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v611452; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc11453; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)11454; GFX11-FAKE16-NEXT: buffer_gl1_inv11455; GFX11-FAKE16-NEXT: buffer_gl0_inv11456; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v611457; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s511458; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11459; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s511460; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB31_111461; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end11462; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x211463; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s511464; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]11465;11466; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:11467; GFX10: ; %bb.0:11468; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11469; GFX10-NEXT: v_mov_b32_e32 v1, v011470; GFX10-NEXT: v_mov_b32_e32 v0, s2011471; GFX10-NEXT: v_mov_b32_e32 v4, s2011472; GFX10-NEXT: s_mov_b32 s5, 011473; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v111474; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102411475; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v111476; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start11477; GFX10-NEXT: ; =>This Inner Loop Header: Depth=111478; GFX10-NEXT: s_waitcnt vmcnt(0)11479; GFX10-NEXT: v_mov_b32_e32 v6, v011480; GFX10-NEXT: s_waitcnt_vscnt null, 0x011481; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v611482; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v611483; GFX10-NEXT: v_add_f32_e32 v0, v0, v211484; GFX10-NEXT: v_add_f32_e32 v1, v1, v311485; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 111486; GFX10-NEXT: v_bfe_u32 v7, v1, 16, 111487; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v011488; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v111489; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v111490; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff11491; GFX10-NEXT: v_add3_u32 v7, v7, v1, 0x7fff11492; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v011493; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo11494; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v8, s411495; GFX10-NEXT: v_perm_b32 v5, v1, v0, 0x706030211496; GFX10-NEXT: v_mov_b32_e32 v0, v511497; GFX10-NEXT: v_mov_b32_e32 v1, v611498; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc11499; GFX10-NEXT: s_waitcnt vmcnt(0)11500; GFX10-NEXT: buffer_gl1_inv11501; GFX10-NEXT: buffer_gl0_inv11502; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v611503; GFX10-NEXT: s_or_b32 s5, vcc_lo, s511504; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s511505; GFX10-NEXT: s_cbranch_execnz .LBB31_111506; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end11507; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s511508; GFX10-NEXT: s_setpc_b64 s[30:31]11509;11510; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:11511; GFX90A: ; %bb.0:11512; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11513; GFX90A-NEXT: v_mov_b32_e32 v1, v011514; GFX90A-NEXT: v_mov_b32_e32 v0, s2011515; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102411516; GFX90A-NEXT: s_mov_b64 s[6:7], 011517; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v111518; GFX90A-NEXT: s_movk_i32 s8, 0x7fff11519; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v111520; GFX90A-NEXT: s_mov_b32 s9, 0x706030211521; GFX90A-NEXT: v_mov_b32_e32 v4, s2011522; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start11523; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111524; GFX90A-NEXT: s_waitcnt vmcnt(0)11525; GFX90A-NEXT: v_mov_b32_e32 v7, v011526; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v711527; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v711528; GFX90A-NEXT: v_add_f32_e32 v0, v0, v211529; GFX90A-NEXT: v_add_f32_e32 v1, v1, v311530; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 111531; GFX90A-NEXT: v_bfe_u32 v8, v1, 16, 111532; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v011533; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v111534; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s811535; GFX90A-NEXT: v_add3_u32 v8, v8, v1, s811536; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v111537; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v011538; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]11539; GFX90A-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc11540; GFX90A-NEXT: v_perm_b32 v6, v1, v0, s911541; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]11542; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc11543; GFX90A-NEXT: s_waitcnt vmcnt(0)11544; GFX90A-NEXT: buffer_wbinvl111545; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v711546; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11547; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]11548; GFX90A-NEXT: s_cbranch_execnz .LBB31_111549; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11550; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]11551; GFX90A-NEXT: s_setpc_b64 s[30:31]11552;11553; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:11554; GFX908: ; %bb.0:11555; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11556; GFX908-NEXT: v_mov_b32_e32 v1, v011557; GFX908-NEXT: v_mov_b32_e32 v0, s2011558; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102411559; GFX908-NEXT: s_mov_b64 s[6:7], 011560; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v111561; GFX908-NEXT: s_movk_i32 s8, 0x7fff11562; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v111563; GFX908-NEXT: s_mov_b32 s9, 0x706030211564; GFX908-NEXT: v_mov_b32_e32 v4, s2011565; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start11566; GFX908-NEXT: ; =>This Inner Loop Header: Depth=111567; GFX908-NEXT: s_waitcnt vmcnt(0)11568; GFX908-NEXT: v_mov_b32_e32 v6, v011569; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v611570; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v611571; GFX908-NEXT: v_add_f32_e32 v0, v0, v211572; GFX908-NEXT: v_add_f32_e32 v1, v1, v311573; GFX908-NEXT: v_bfe_u32 v5, v0, 16, 111574; GFX908-NEXT: v_bfe_u32 v8, v1, 16, 111575; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v011576; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v111577; GFX908-NEXT: v_add3_u32 v5, v5, v0, s811578; GFX908-NEXT: v_add3_u32 v8, v8, v1, s811579; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v111580; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v011581; GFX908-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]11582; GFX908-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc11583; GFX908-NEXT: v_perm_b32 v5, v1, v0, s911584; GFX908-NEXT: v_mov_b32_e32 v0, v511585; GFX908-NEXT: v_mov_b32_e32 v1, v611586; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc11587; GFX908-NEXT: s_waitcnt vmcnt(0)11588; GFX908-NEXT: buffer_wbinvl111589; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v611590; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11591; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]11592; GFX908-NEXT: s_cbranch_execnz .LBB31_111593; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end11594; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]11595; GFX908-NEXT: s_setpc_b64 s[30:31]11596;11597; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:11598; GFX8: ; %bb.0:11599; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11600; GFX8-NEXT: v_mov_b32_e32 v1, v011601; GFX8-NEXT: v_mov_b32_e32 v0, s2011602; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102411603; GFX8-NEXT: s_mov_b64 s[6:7], 011604; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v111605; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v111606; GFX8-NEXT: v_mov_b32_e32 v4, s2011607; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start11608; GFX8-NEXT: ; =>This Inner Loop Header: Depth=111609; GFX8-NEXT: s_waitcnt vmcnt(0)11610; GFX8-NEXT: v_mov_b32_e32 v6, v011611; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v611612; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v611613; GFX8-NEXT: v_add_f32_e32 v0, v0, v211614; GFX8-NEXT: v_add_f32_e32 v1, v1, v311615; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 111616; GFX8-NEXT: v_bfe_u32 v8, v1, 16, 111617; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v011618; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v111619; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v511620; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v811621; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v111622; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v111623; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v011624; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v011625; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc11626; GFX8-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]11627; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v111628; GFX8-NEXT: v_alignbit_b32 v5, v1, v0, 1611629; GFX8-NEXT: v_mov_b32_e32 v0, v511630; GFX8-NEXT: v_mov_b32_e32 v1, v611631; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc11632; GFX8-NEXT: s_waitcnt vmcnt(0)11633; GFX8-NEXT: buffer_wbinvl111634; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v611635; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11636; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]11637; GFX8-NEXT: s_cbranch_execnz .LBB31_111638; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end11639; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]11640; GFX8-NEXT: s_setpc_b64 s[30:31]11641;11642; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:11643; GFX7: ; %bb.0:11644; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11645; GFX7-NEXT: v_mov_b32_e32 v2, s2011646; GFX7-NEXT: buffer_load_dword v4, v2, s[16:19], 0 offen offset:102411647; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v011648; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v111649; GFX7-NEXT: s_mov_b64 s[4:5], 011650; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v011651; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v111652; GFX7-NEXT: s_waitcnt vmcnt(0)11653; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v411654; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v411655; GFX7-NEXT: v_mov_b32_e32 v4, s2011656; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start11657; GFX7-NEXT: ; =>This Inner Loop Header: Depth=111658; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v111659; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v011660; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v111661; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v011662; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v111663; GFX7-NEXT: v_add_f32_e32 v6, v6, v311664; GFX7-NEXT: v_add_f32_e32 v5, v5, v211665; GFX7-NEXT: v_alignbit_b32 v1, v1, v0, 1611666; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v611667; GFX7-NEXT: v_alignbit_b32 v0, v0, v5, 1611668; GFX7-NEXT: v_mov_b32_e32 v6, v111669; GFX7-NEXT: v_mov_b32_e32 v5, v011670; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc11671; GFX7-NEXT: s_waitcnt vmcnt(0)11672; GFX7-NEXT: buffer_wbinvl111673; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v111674; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v511675; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11676; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v511677; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]11678; GFX7-NEXT: s_cbranch_execnz .LBB31_111679; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end11680; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]11681; GFX7-NEXT: s_setpc_b64 s[30:31]11682;11683; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:11684; GFX6: ; %bb.0:11685; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11686; GFX6-NEXT: v_mov_b32_e32 v2, s2011687; GFX6-NEXT: buffer_load_dword v4, v2, s[16:19], 0 offen offset:102411688; GFX6-NEXT: s_add_i32 s6, s20, 0x40011689; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v011690; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v111691; GFX6-NEXT: s_mov_b64 s[4:5], 011692; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v011693; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v111694; GFX6-NEXT: s_waitcnt vmcnt(0)11695; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v411696; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v411697; GFX6-NEXT: v_mov_b32_e32 v4, s611698; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start11699; GFX6-NEXT: ; =>This Inner Loop Header: Depth=111700; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v111701; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v011702; GFX6-NEXT: s_waitcnt expcnt(0)11703; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v111704; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v011705; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v111706; GFX6-NEXT: v_add_f32_e32 v6, v6, v311707; GFX6-NEXT: v_add_f32_e32 v5, v5, v211708; GFX6-NEXT: v_alignbit_b32 v1, v1, v0, 1611709; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v611710; GFX6-NEXT: v_alignbit_b32 v0, v0, v5, 1611711; GFX6-NEXT: v_mov_b32_e32 v6, v111712; GFX6-NEXT: v_mov_b32_e32 v5, v011713; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen glc11714; GFX6-NEXT: s_waitcnt vmcnt(0)11715; GFX6-NEXT: buffer_wbinvl111716; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v111717; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v511718; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]11719; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v511720; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]11721; GFX6-NEXT: s_cbranch_execnz .LBB31_111722; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end11723; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]11724; GFX6-NEXT: s_waitcnt expcnt(0)11725; GFX6-NEXT: s_setpc_b64 s[30:31]11726 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 25611727 %result = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !011728 ret <2 x bfloat> %result11729}11730 11731define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {11732; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:11733; GFX12: ; %bb.0:11734; GFX12-NEXT: s_wait_loadcnt_dscnt 0x011735; GFX12-NEXT: s_wait_expcnt 0x011736; GFX12-NEXT: s_wait_samplecnt 0x011737; GFX12-NEXT: s_wait_bvhcnt 0x011738; GFX12-NEXT: s_wait_kmcnt 0x011739; GFX12-NEXT: v_mov_b32_e32 v1, s1611740; GFX12-NEXT: s_wait_storecnt 0x011741; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:102411742; GFX12-NEXT: s_wait_storecnt 0x011743; GFX12-NEXT: global_inv scope:SCOPE_DEV11744; GFX12-NEXT: s_wait_loadcnt 0x011745; GFX12-NEXT: s_setpc_b64 s[30:31]11746;11747; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:11748; GFX942: ; %bb.0:11749; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11750; GFX942-NEXT: v_mov_b32_e32 v1, s1611751; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:102411752; GFX942-NEXT: s_mov_b64 s[6:7], 011753; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v011754; GFX942-NEXT: s_movk_i32 s8, 0x7fff11755; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v011756; GFX942-NEXT: s_mov_b32 s9, 0x706030211757; GFX942-NEXT: v_mov_b32_e32 v4, s1611758; GFX942-NEXT: .LBB32_1: ; %atomicrmw.start11759; GFX942-NEXT: ; =>This Inner Loop Header: Depth=111760; GFX942-NEXT: s_waitcnt vmcnt(0)11761; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v111762; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v111763; GFX942-NEXT: v_add_f32_e32 v0, v0, v211764; GFX942-NEXT: v_add_f32_e32 v5, v5, v311765; GFX942-NEXT: v_bfe_u32 v6, v0, 16, 111766; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 111767; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v011768; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v511769; GFX942-NEXT: v_add3_u32 v6, v6, v0, s811770; GFX942-NEXT: v_add3_u32 v8, v8, v5, s811771; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v511772; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v011773; GFX942-NEXT: buffer_wbl2 sc111774; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc11775; GFX942-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]11776; GFX942-NEXT: v_perm_b32 v0, v5, v0, s911777; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1]11778; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen offset:1024 sc011779; GFX942-NEXT: s_waitcnt vmcnt(0)11780; GFX942-NEXT: buffer_inv sc111781; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v111782; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11783; GFX942-NEXT: v_mov_b32_e32 v1, v611784; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]11785; GFX942-NEXT: s_cbranch_execnz .LBB32_111786; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end11787; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]11788; GFX942-NEXT: s_setpc_b64 s[30:31]11789;11790; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:11791; GFX11-TRUE16: ; %bb.0:11792; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11793; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v011794; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v011795; GFX11-TRUE16-NEXT: s_mov_b32 s4, 011796; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:102411797; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x111798; GFX11-TRUE16-NEXT: .p2align 611799; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start11800; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=111801; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)11802; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v111803; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v111804; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x011805; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11806; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v2 :: v_dual_add_f32 v0, v0, v311807; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 111808; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)11809; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 111810; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v011811; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v011812; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v511813; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff11814; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff11815; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)11816; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo11817; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v511818; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo11819; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)11820; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h11821; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v011822; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc11823; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)11824; GFX11-TRUE16-NEXT: buffer_gl1_inv11825; GFX11-TRUE16-NEXT: buffer_gl0_inv11826; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v111827; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v511828; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s411829; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11830; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s411831; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_111832; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end11833; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x211834; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s411835; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]11836;11837; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:11838; GFX11-FAKE16: ; %bb.0:11839; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11840; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v011841; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v011842; GFX11-FAKE16-NEXT: s_mov_b32 s5, 011843; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:102411844; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x111845; GFX11-FAKE16-NEXT: .p2align 611846; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start11847; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=111848; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)11849; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v111850; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v111851; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x011852; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11853; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v211854; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 111855; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)11856; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 111857; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v011858; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v511859; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v511860; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff11861; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff11862; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v011863; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)11864; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo11865; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s411866; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)11867; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x706030211868; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v011869; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc11870; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)11871; GFX11-FAKE16-NEXT: buffer_gl1_inv11872; GFX11-FAKE16-NEXT: buffer_gl0_inv11873; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v111874; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v511875; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s511876; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)11877; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s511878; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_111879; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end11880; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x211881; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s511882; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]11883;11884; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:11885; GFX10: ; %bb.0:11886; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11887; GFX10-NEXT: v_mov_b32_e32 v1, s2011888; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v011889; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v011890; GFX10-NEXT: v_mov_b32_e32 v4, s2011891; GFX10-NEXT: s_mov_b32 s5, 011892; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102411893; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start11894; GFX10-NEXT: ; =>This Inner Loop Header: Depth=111895; GFX10-NEXT: s_waitcnt vmcnt(0)11896; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v111897; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v111898; GFX10-NEXT: s_waitcnt_vscnt null, 0x011899; GFX10-NEXT: v_add_f32_e32 v0, v0, v211900; GFX10-NEXT: v_add_f32_e32 v5, v5, v311901; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 111902; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 111903; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v011904; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v511905; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v511906; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff11907; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff11908; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v011909; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo11910; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s411911; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x706030211912; GFX10-NEXT: v_mov_b32_e32 v6, v111913; GFX10-NEXT: v_mov_b32_e32 v5, v011914; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc11915; GFX10-NEXT: s_waitcnt vmcnt(0)11916; GFX10-NEXT: buffer_gl1_inv11917; GFX10-NEXT: buffer_gl0_inv11918; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v111919; GFX10-NEXT: v_mov_b32_e32 v1, v511920; GFX10-NEXT: s_or_b32 s5, vcc_lo, s511921; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s511922; GFX10-NEXT: s_cbranch_execnz .LBB32_111923; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end11924; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s511925; GFX10-NEXT: s_setpc_b64 s[30:31]11926;11927; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:11928; GFX90A: ; %bb.0:11929; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11930; GFX90A-NEXT: v_mov_b32_e32 v1, s2011931; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102411932; GFX90A-NEXT: s_mov_b64 s[6:7], 011933; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v011934; GFX90A-NEXT: s_movk_i32 s8, 0x7fff11935; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v011936; GFX90A-NEXT: s_mov_b32 s9, 0x706030211937; GFX90A-NEXT: v_mov_b32_e32 v4, s2011938; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start11939; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=111940; GFX90A-NEXT: s_waitcnt vmcnt(0)11941; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v111942; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v111943; GFX90A-NEXT: v_add_f32_e32 v0, v0, v211944; GFX90A-NEXT: v_add_f32_e32 v5, v5, v311945; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 111946; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 111947; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v011948; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v511949; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s811950; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s811951; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v511952; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v011953; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]11954; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc11955; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s911956; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]11957; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen offset:1024 glc11958; GFX90A-NEXT: s_waitcnt vmcnt(0)11959; GFX90A-NEXT: buffer_wbinvl111960; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v111961; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]11962; GFX90A-NEXT: v_mov_b32_e32 v1, v611963; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]11964; GFX90A-NEXT: s_cbranch_execnz .LBB32_111965; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end11966; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]11967; GFX90A-NEXT: s_setpc_b64 s[30:31]11968;11969; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:11970; GFX908: ; %bb.0:11971; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11972; GFX908-NEXT: v_mov_b32_e32 v1, s2011973; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102411974; GFX908-NEXT: s_mov_b64 s[6:7], 011975; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v011976; GFX908-NEXT: s_movk_i32 s8, 0x7fff11977; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v011978; GFX908-NEXT: s_mov_b32 s9, 0x706030211979; GFX908-NEXT: v_mov_b32_e32 v4, s2011980; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start11981; GFX908-NEXT: ; =>This Inner Loop Header: Depth=111982; GFX908-NEXT: s_waitcnt vmcnt(0)11983; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v111984; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v111985; GFX908-NEXT: v_add_f32_e32 v0, v0, v211986; GFX908-NEXT: v_add_f32_e32 v5, v5, v311987; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 111988; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 111989; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v011990; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v511991; GFX908-NEXT: v_add3_u32 v6, v6, v0, s811992; GFX908-NEXT: v_add3_u32 v8, v8, v5, s811993; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v511994; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v011995; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]11996; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc11997; GFX908-NEXT: v_perm_b32 v0, v5, v0, s911998; GFX908-NEXT: v_mov_b32_e32 v6, v111999; GFX908-NEXT: v_mov_b32_e32 v5, v012000; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc12001; GFX908-NEXT: s_waitcnt vmcnt(0)12002; GFX908-NEXT: buffer_wbinvl112003; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v112004; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12005; GFX908-NEXT: v_mov_b32_e32 v1, v512006; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]12007; GFX908-NEXT: s_cbranch_execnz .LBB32_112008; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end12009; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]12010; GFX908-NEXT: s_setpc_b64 s[30:31]12011;12012; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:12013; GFX8: ; %bb.0:12014; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12015; GFX8-NEXT: v_mov_b32_e32 v1, s2012016; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102412017; GFX8-NEXT: s_mov_b64 s[6:7], 012018; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v012019; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v012020; GFX8-NEXT: v_mov_b32_e32 v4, s2012021; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start12022; GFX8-NEXT: ; =>This Inner Loop Header: Depth=112023; GFX8-NEXT: s_waitcnt vmcnt(0)12024; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v112025; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v112026; GFX8-NEXT: v_add_f32_e32 v0, v0, v212027; GFX8-NEXT: v_add_f32_e32 v5, v5, v312028; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 112029; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 112030; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v012031; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v512032; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v612033; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v812034; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v512035; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v512036; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v012037; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v012038; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc12039; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]12040; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v512041; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 1612042; GFX8-NEXT: v_mov_b32_e32 v6, v112043; GFX8-NEXT: v_mov_b32_e32 v5, v012044; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc12045; GFX8-NEXT: s_waitcnt vmcnt(0)12046; GFX8-NEXT: buffer_wbinvl112047; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v112048; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12049; GFX8-NEXT: v_mov_b32_e32 v1, v512050; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]12051; GFX8-NEXT: s_cbranch_execnz .LBB32_112052; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end12053; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]12054; GFX8-NEXT: s_setpc_b64 s[30:31]12055;12056; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:12057; GFX7: ; %bb.0:12058; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12059; GFX7-NEXT: v_mov_b32_e32 v2, s2012060; GFX7-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:102412061; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v012062; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v112063; GFX7-NEXT: s_mov_b64 s[4:5], 012064; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v012065; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v112066; GFX7-NEXT: s_waitcnt vmcnt(0)12067; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v212068; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v212069; GFX7-NEXT: v_mov_b32_e32 v2, s2012070; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start12071; GFX7-NEXT: ; =>This Inner Loop Header: Depth=112072; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v312073; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v412074; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v312075; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v412076; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v312077; GFX7-NEXT: v_add_f32_e32 v6, v6, v112078; GFX7-NEXT: v_add_f32_e32 v5, v5, v012079; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 1612080; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v612081; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 1612082; GFX7-NEXT: v_mov_b32_e32 v6, v412083; GFX7-NEXT: v_mov_b32_e32 v5, v312084; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc12085; GFX7-NEXT: s_waitcnt vmcnt(0)12086; GFX7-NEXT: buffer_wbinvl112087; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v412088; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v512089; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]12090; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v512091; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]12092; GFX7-NEXT: s_cbranch_execnz .LBB32_112093; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end12094; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]12095; GFX7-NEXT: s_setpc_b64 s[30:31]12096;12097; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:12098; GFX6: ; %bb.0:12099; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12100; GFX6-NEXT: v_mov_b32_e32 v2, s2012101; GFX6-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:102412102; GFX6-NEXT: s_add_i32 s6, s20, 0x40012103; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v012104; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v112105; GFX6-NEXT: s_mov_b64 s[4:5], 012106; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v012107; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v112108; GFX6-NEXT: s_waitcnt vmcnt(0)12109; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v212110; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v212111; GFX6-NEXT: v_mov_b32_e32 v2, s612112; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start12113; GFX6-NEXT: ; =>This Inner Loop Header: Depth=112114; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v312115; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v412116; GFX6-NEXT: s_waitcnt expcnt(0)12117; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v312118; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v412119; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v312120; GFX6-NEXT: v_add_f32_e32 v6, v6, v112121; GFX6-NEXT: v_add_f32_e32 v5, v5, v012122; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 1612123; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v612124; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 1612125; GFX6-NEXT: v_mov_b32_e32 v6, v412126; GFX6-NEXT: v_mov_b32_e32 v5, v312127; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc12128; GFX6-NEXT: s_waitcnt vmcnt(0)12129; GFX6-NEXT: buffer_wbinvl112130; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v412131; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v512132; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]12133; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v512134; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]12135; GFX6-NEXT: s_cbranch_execnz .LBB32_112136; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end12137; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]12138; GFX6-NEXT: s_waitcnt expcnt(0)12139; GFX6-NEXT: s_setpc_b64 s[30:31]12140 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 25612141 %unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !012142 ret void12143}12144 12145define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(7) inreg %ptr, <2 x bfloat> %val) #0 {12146; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:12147; GFX12: ; %bb.0:12148; GFX12-NEXT: s_wait_loadcnt_dscnt 0x012149; GFX12-NEXT: s_wait_expcnt 0x012150; GFX12-NEXT: s_wait_samplecnt 0x012151; GFX12-NEXT: s_wait_bvhcnt 0x012152; GFX12-NEXT: s_wait_kmcnt 0x012153; GFX12-NEXT: v_mov_b32_e32 v1, s1612154; GFX12-NEXT: s_wait_storecnt 0x012155; GFX12-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], null offen offset:102412156; GFX12-NEXT: s_wait_storecnt 0x012157; GFX12-NEXT: global_inv scope:SCOPE_DEV12158; GFX12-NEXT: s_wait_loadcnt 0x012159; GFX12-NEXT: s_setpc_b64 s[30:31]12160;12161; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:12162; GFX942: ; %bb.0:12163; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12164; GFX942-NEXT: v_mov_b32_e32 v1, s1612165; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:102412166; GFX942-NEXT: s_mov_b64 s[6:7], 012167; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v012168; GFX942-NEXT: s_movk_i32 s8, 0x7fff12169; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v012170; GFX942-NEXT: s_mov_b32 s9, 0x706030212171; GFX942-NEXT: v_mov_b32_e32 v4, s1612172; GFX942-NEXT: .LBB33_1: ; %atomicrmw.start12173; GFX942-NEXT: ; =>This Inner Loop Header: Depth=112174; GFX942-NEXT: s_waitcnt vmcnt(0)12175; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v112176; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v112177; GFX942-NEXT: v_add_f32_e32 v0, v0, v212178; GFX942-NEXT: v_add_f32_e32 v5, v5, v312179; GFX942-NEXT: v_bfe_u32 v6, v0, 16, 112180; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 112181; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v012182; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v512183; GFX942-NEXT: v_add3_u32 v6, v6, v0, s812184; GFX942-NEXT: v_add3_u32 v8, v8, v5, s812185; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v512186; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v012187; GFX942-NEXT: buffer_wbl2 sc112188; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc12189; GFX942-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]12190; GFX942-NEXT: v_perm_b32 v0, v5, v0, s912191; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1]12192; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen offset:1024 sc012193; GFX942-NEXT: s_waitcnt vmcnt(0)12194; GFX942-NEXT: buffer_inv sc112195; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v112196; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12197; GFX942-NEXT: v_mov_b32_e32 v1, v612198; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]12199; GFX942-NEXT: s_cbranch_execnz .LBB33_112200; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end12201; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]12202; GFX942-NEXT: s_setpc_b64 s[30:31]12203;12204; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:12205; GFX11-TRUE16: ; %bb.0:12206; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12207; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v012208; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_lshlrev_b32 v3, 16, v012209; GFX11-TRUE16-NEXT: s_mov_b32 s4, 012210; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:102412211; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x112212; GFX11-TRUE16-NEXT: .p2align 612213; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start12214; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=112215; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)12216; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v112217; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v112218; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x012219; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12220; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v2 :: v_dual_add_f32 v0, v0, v312221; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 112222; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)12223; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 112224; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v012225; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v012226; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v512227; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff12228; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff12229; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)12230; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo12231; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v512232; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo12233; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)12234; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h12235; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v012236; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc12237; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)12238; GFX11-TRUE16-NEXT: buffer_gl1_inv12239; GFX11-TRUE16-NEXT: buffer_gl0_inv12240; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v112241; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v512242; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s412243; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12244; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s412245; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_112246; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end12247; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x212248; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s412249; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]12250;12251; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:12252; GFX11-FAKE16: ; %bb.0:12253; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12254; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_lshlrev_b32 v2, 16, v012255; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_and_b32 v3, 0xffff0000, v012256; GFX11-FAKE16-NEXT: s_mov_b32 s5, 012257; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen offset:102412258; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x112259; GFX11-FAKE16-NEXT: .p2align 612260; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start12261; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=112262; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)12263; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v112264; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v112265; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x012266; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12267; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v212268; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 112269; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)12270; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 112271; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v012272; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v512273; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v512274; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff12275; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff12276; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v012277; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)12278; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo12279; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s412280; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)12281; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x706030212282; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v012283; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc12284; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)12285; GFX11-FAKE16-NEXT: buffer_gl1_inv12286; GFX11-FAKE16-NEXT: buffer_gl0_inv12287; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v112288; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v512289; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s512290; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)12291; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s512292; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_112293; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end12294; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x212295; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s512296; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]12297;12298; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:12299; GFX10: ; %bb.0:12300; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12301; GFX10-NEXT: v_mov_b32_e32 v1, s2012302; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v012303; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v012304; GFX10-NEXT: v_mov_b32_e32 v4, s2012305; GFX10-NEXT: s_mov_b32 s5, 012306; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102412307; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start12308; GFX10-NEXT: ; =>This Inner Loop Header: Depth=112309; GFX10-NEXT: s_waitcnt vmcnt(0)12310; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v112311; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v112312; GFX10-NEXT: s_waitcnt_vscnt null, 0x012313; GFX10-NEXT: v_add_f32_e32 v0, v0, v212314; GFX10-NEXT: v_add_f32_e32 v5, v5, v312315; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 112316; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 112317; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v012318; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v512319; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v512320; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff12321; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff12322; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v012323; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo12324; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s412325; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x706030212326; GFX10-NEXT: v_mov_b32_e32 v6, v112327; GFX10-NEXT: v_mov_b32_e32 v5, v012328; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc12329; GFX10-NEXT: s_waitcnt vmcnt(0)12330; GFX10-NEXT: buffer_gl1_inv12331; GFX10-NEXT: buffer_gl0_inv12332; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v112333; GFX10-NEXT: v_mov_b32_e32 v1, v512334; GFX10-NEXT: s_or_b32 s5, vcc_lo, s512335; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s512336; GFX10-NEXT: s_cbranch_execnz .LBB33_112337; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end12338; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s512339; GFX10-NEXT: s_setpc_b64 s[30:31]12340;12341; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:12342; GFX90A: ; %bb.0:12343; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12344; GFX90A-NEXT: v_mov_b32_e32 v1, s2012345; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102412346; GFX90A-NEXT: s_mov_b64 s[6:7], 012347; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v012348; GFX90A-NEXT: s_movk_i32 s8, 0x7fff12349; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v012350; GFX90A-NEXT: s_mov_b32 s9, 0x706030212351; GFX90A-NEXT: v_mov_b32_e32 v4, s2012352; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start12353; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112354; GFX90A-NEXT: s_waitcnt vmcnt(0)12355; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v112356; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v112357; GFX90A-NEXT: v_add_f32_e32 v0, v0, v212358; GFX90A-NEXT: v_add_f32_e32 v5, v5, v312359; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 112360; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 112361; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v012362; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v512363; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s812364; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s812365; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v512366; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v012367; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]12368; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc12369; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s912370; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]12371; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen offset:1024 glc12372; GFX90A-NEXT: s_waitcnt vmcnt(0)12373; GFX90A-NEXT: buffer_wbinvl112374; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v112375; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12376; GFX90A-NEXT: v_mov_b32_e32 v1, v612377; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]12378; GFX90A-NEXT: s_cbranch_execnz .LBB33_112379; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12380; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]12381; GFX90A-NEXT: s_setpc_b64 s[30:31]12382;12383; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:12384; GFX908: ; %bb.0:12385; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12386; GFX908-NEXT: v_mov_b32_e32 v1, s2012387; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102412388; GFX908-NEXT: s_mov_b64 s[6:7], 012389; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v012390; GFX908-NEXT: s_movk_i32 s8, 0x7fff12391; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v012392; GFX908-NEXT: s_mov_b32 s9, 0x706030212393; GFX908-NEXT: v_mov_b32_e32 v4, s2012394; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start12395; GFX908-NEXT: ; =>This Inner Loop Header: Depth=112396; GFX908-NEXT: s_waitcnt vmcnt(0)12397; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v112398; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v112399; GFX908-NEXT: v_add_f32_e32 v0, v0, v212400; GFX908-NEXT: v_add_f32_e32 v5, v5, v312401; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 112402; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 112403; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v012404; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v512405; GFX908-NEXT: v_add3_u32 v6, v6, v0, s812406; GFX908-NEXT: v_add3_u32 v8, v8, v5, s812407; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v512408; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v012409; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]12410; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc12411; GFX908-NEXT: v_perm_b32 v0, v5, v0, s912412; GFX908-NEXT: v_mov_b32_e32 v6, v112413; GFX908-NEXT: v_mov_b32_e32 v5, v012414; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc12415; GFX908-NEXT: s_waitcnt vmcnt(0)12416; GFX908-NEXT: buffer_wbinvl112417; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v112418; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12419; GFX908-NEXT: v_mov_b32_e32 v1, v512420; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]12421; GFX908-NEXT: s_cbranch_execnz .LBB33_112422; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end12423; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]12424; GFX908-NEXT: s_setpc_b64 s[30:31]12425;12426; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:12427; GFX8: ; %bb.0:12428; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12429; GFX8-NEXT: v_mov_b32_e32 v1, s2012430; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:102412431; GFX8-NEXT: s_mov_b64 s[6:7], 012432; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v012433; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v012434; GFX8-NEXT: v_mov_b32_e32 v4, s2012435; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start12436; GFX8-NEXT: ; =>This Inner Loop Header: Depth=112437; GFX8-NEXT: s_waitcnt vmcnt(0)12438; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v112439; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v112440; GFX8-NEXT: v_add_f32_e32 v0, v0, v212441; GFX8-NEXT: v_add_f32_e32 v5, v5, v312442; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 112443; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 112444; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v012445; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v512446; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v612447; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v812448; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v512449; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v512450; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v012451; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v012452; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc12453; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]12454; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v512455; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 1612456; GFX8-NEXT: v_mov_b32_e32 v6, v112457; GFX8-NEXT: v_mov_b32_e32 v5, v012458; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc12459; GFX8-NEXT: s_waitcnt vmcnt(0)12460; GFX8-NEXT: buffer_wbinvl112461; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v112462; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]12463; GFX8-NEXT: v_mov_b32_e32 v1, v512464; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]12465; GFX8-NEXT: s_cbranch_execnz .LBB33_112466; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end12467; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]12468; GFX8-NEXT: s_setpc_b64 s[30:31]12469;12470; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:12471; GFX7: ; %bb.0:12472; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12473; GFX7-NEXT: v_mov_b32_e32 v2, s2012474; GFX7-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:102412475; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v012476; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v112477; GFX7-NEXT: s_mov_b64 s[4:5], 012478; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v012479; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v112480; GFX7-NEXT: s_waitcnt vmcnt(0)12481; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v212482; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v212483; GFX7-NEXT: v_mov_b32_e32 v2, s2012484; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start12485; GFX7-NEXT: ; =>This Inner Loop Header: Depth=112486; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v312487; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v412488; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v312489; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v412490; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v312491; GFX7-NEXT: v_add_f32_e32 v6, v6, v112492; GFX7-NEXT: v_add_f32_e32 v5, v5, v012493; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 1612494; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v612495; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 1612496; GFX7-NEXT: v_mov_b32_e32 v6, v412497; GFX7-NEXT: v_mov_b32_e32 v5, v312498; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc12499; GFX7-NEXT: s_waitcnt vmcnt(0)12500; GFX7-NEXT: buffer_wbinvl112501; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v412502; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v512503; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]12504; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v512505; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]12506; GFX7-NEXT: s_cbranch_execnz .LBB33_112507; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end12508; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]12509; GFX7-NEXT: s_setpc_b64 s[30:31]12510;12511; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:12512; GFX6: ; %bb.0:12513; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12514; GFX6-NEXT: v_mov_b32_e32 v2, s2012515; GFX6-NEXT: buffer_load_dword v2, v2, s[16:19], 0 offen offset:102412516; GFX6-NEXT: s_add_i32 s6, s20, 0x40012517; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v012518; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v112519; GFX6-NEXT: s_mov_b64 s[4:5], 012520; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v012521; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v112522; GFX6-NEXT: s_waitcnt vmcnt(0)12523; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v212524; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v212525; GFX6-NEXT: v_mov_b32_e32 v2, s612526; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start12527; GFX6-NEXT: ; =>This Inner Loop Header: Depth=112528; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v312529; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v412530; GFX6-NEXT: s_waitcnt expcnt(0)12531; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v312532; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v412533; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v312534; GFX6-NEXT: v_add_f32_e32 v6, v6, v112535; GFX6-NEXT: v_add_f32_e32 v5, v5, v012536; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 1612537; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v612538; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 1612539; GFX6-NEXT: v_mov_b32_e32 v6, v412540; GFX6-NEXT: v_mov_b32_e32 v5, v312541; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc12542; GFX6-NEXT: s_waitcnt vmcnt(0)12543; GFX6-NEXT: buffer_wbinvl112544; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v412545; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v512546; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]12547; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v512548; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]12549; GFX6-NEXT: s_cbranch_execnz .LBB33_112550; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end12551; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]12552; GFX6-NEXT: s_waitcnt expcnt(0)12553; GFX6-NEXT: s_setpc_b64 s[30:31]12554 %gep = getelementptr <2 x bfloat>, ptr addrspace(7) %ptr, i32 25612555 %unused = atomicrmw fadd ptr addrspace(7) %gep, <2 x bfloat> %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !012556 ret void12557}12558 12559; --------------------------------------------------------------------12560; misc12561; --------------------------------------------------------------------12562 12563define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, float %val) #0 {12564; GFX12-LABEL: buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:12565; GFX12: ; %bb.0:12566; GFX12-NEXT: s_wait_loadcnt_dscnt 0x012567; GFX12-NEXT: s_wait_expcnt 0x012568; GFX12-NEXT: s_wait_samplecnt 0x012569; GFX12-NEXT: s_wait_bvhcnt 0x012570; GFX12-NEXT: s_wait_kmcnt 0x012571; GFX12-NEXT: v_mov_b32_e32 v1, s1612572; GFX12-NEXT: global_wb scope:SCOPE_SYS12573; GFX12-NEXT: s_wait_storecnt 0x012574; GFX12-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN12575; GFX12-NEXT: s_wait_loadcnt 0x012576; GFX12-NEXT: global_inv scope:SCOPE_SYS12577; GFX12-NEXT: s_wait_loadcnt 0x012578; GFX12-NEXT: s_setpc_b64 s[30:31]12579;12580; GFX942-LABEL: buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:12581; GFX942: ; %bb.0:12582; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12583; GFX942-NEXT: v_mov_b32_e32 v1, s1612584; GFX942-NEXT: buffer_wbl2 sc0 sc112585; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 sc012586; GFX942-NEXT: s_waitcnt vmcnt(0)12587; GFX942-NEXT: buffer_inv sc0 sc112588; GFX942-NEXT: s_setpc_b64 s[30:31]12589;12590; GFX11-LABEL: buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:12591; GFX11: ; %bb.0:12592; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12593; GFX11-NEXT: v_mov_b32_e32 v1, s1612594; GFX11-NEXT: s_waitcnt_vscnt null, 0x012595; GFX11-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], 0 offen offset:1024 glc12596; GFX11-NEXT: s_waitcnt vmcnt(0)12597; GFX11-NEXT: buffer_gl1_inv12598; GFX11-NEXT: buffer_gl0_inv12599; GFX11-NEXT: s_setpc_b64 s[30:31]12600;12601; GFX10-LABEL: buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:12602; GFX10: ; %bb.0:12603; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12604; GFX10-NEXT: v_mov_b32_e32 v2, v012605; GFX10-NEXT: v_mov_b32_e32 v0, s2012606; GFX10-NEXT: v_mov_b32_e32 v3, s2012607; GFX10-NEXT: s_mov_b32 s4, 012608; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102412609; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start12610; GFX10-NEXT: ; =>This Inner Loop Header: Depth=112611; GFX10-NEXT: s_waitcnt vmcnt(0)12612; GFX10-NEXT: v_mov_b32_e32 v5, v012613; GFX10-NEXT: s_waitcnt_vscnt null, 0x012614; GFX10-NEXT: v_add_f32_e32 v4, v5, v212615; GFX10-NEXT: v_mov_b32_e32 v0, v412616; GFX10-NEXT: v_mov_b32_e32 v1, v512617; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc12618; GFX10-NEXT: s_waitcnt vmcnt(0)12619; GFX10-NEXT: buffer_gl1_inv12620; GFX10-NEXT: buffer_gl0_inv12621; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v512622; GFX10-NEXT: s_or_b32 s4, vcc_lo, s412623; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s412624; GFX10-NEXT: s_cbranch_execnz .LBB34_112625; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end12626; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s412627; GFX10-NEXT: s_setpc_b64 s[30:31]12628;12629; GFX90A-LABEL: buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:12630; GFX90A: ; %bb.0:12631; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12632; GFX90A-NEXT: v_mov_b32_e32 v2, v012633; GFX90A-NEXT: v_mov_b32_e32 v0, s2012634; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102412635; GFX90A-NEXT: s_mov_b64 s[4:5], 012636; GFX90A-NEXT: v_mov_b32_e32 v3, s2012637; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start12638; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=112639; GFX90A-NEXT: s_waitcnt vmcnt(0)12640; GFX90A-NEXT: v_mov_b32_e32 v5, v012641; GFX90A-NEXT: v_add_f32_e32 v4, v5, v212642; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]12643; GFX90A-NEXT: buffer_wbl212644; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc12645; GFX90A-NEXT: s_waitcnt vmcnt(0)12646; GFX90A-NEXT: buffer_invl212647; GFX90A-NEXT: buffer_wbinvl112648; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v512649; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]12650; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]12651; GFX90A-NEXT: s_cbranch_execnz .LBB34_112652; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end12653; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]12654; GFX90A-NEXT: s_setpc_b64 s[30:31]12655;12656; GFX908-LABEL: buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:12657; GFX908: ; %bb.0:12658; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12659; GFX908-NEXT: v_mov_b32_e32 v2, v012660; GFX908-NEXT: v_mov_b32_e32 v0, s2012661; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102412662; GFX908-NEXT: s_mov_b64 s[4:5], 012663; GFX908-NEXT: v_mov_b32_e32 v3, s2012664; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start12665; GFX908-NEXT: ; =>This Inner Loop Header: Depth=112666; GFX908-NEXT: s_waitcnt vmcnt(0)12667; GFX908-NEXT: v_mov_b32_e32 v5, v012668; GFX908-NEXT: v_add_f32_e32 v4, v5, v212669; GFX908-NEXT: v_mov_b32_e32 v0, v412670; GFX908-NEXT: v_mov_b32_e32 v1, v512671; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc12672; GFX908-NEXT: s_waitcnt vmcnt(0)12673; GFX908-NEXT: buffer_wbinvl112674; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v512675; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]12676; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]12677; GFX908-NEXT: s_cbranch_execnz .LBB34_112678; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end12679; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]12680; GFX908-NEXT: s_setpc_b64 s[30:31]12681;12682; GFX8-LABEL: buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:12683; GFX8: ; %bb.0:12684; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12685; GFX8-NEXT: v_mov_b32_e32 v2, v012686; GFX8-NEXT: v_mov_b32_e32 v0, s2012687; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102412688; GFX8-NEXT: s_mov_b64 s[4:5], 012689; GFX8-NEXT: v_mov_b32_e32 v3, s2012690; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start12691; GFX8-NEXT: ; =>This Inner Loop Header: Depth=112692; GFX8-NEXT: s_waitcnt vmcnt(0)12693; GFX8-NEXT: v_mov_b32_e32 v5, v012694; GFX8-NEXT: v_add_f32_e32 v4, v5, v212695; GFX8-NEXT: v_mov_b32_e32 v0, v412696; GFX8-NEXT: v_mov_b32_e32 v1, v512697; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc12698; GFX8-NEXT: s_waitcnt vmcnt(0)12699; GFX8-NEXT: buffer_wbinvl112700; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v512701; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]12702; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]12703; GFX8-NEXT: s_cbranch_execnz .LBB34_112704; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end12705; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]12706; GFX8-NEXT: s_setpc_b64 s[30:31]12707;12708; GFX7-LABEL: buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:12709; GFX7: ; %bb.0:12710; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12711; GFX7-NEXT: v_mov_b32_e32 v2, v012712; GFX7-NEXT: v_mov_b32_e32 v0, s2012713; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102412714; GFX7-NEXT: s_mov_b64 s[4:5], 012715; GFX7-NEXT: v_mov_b32_e32 v3, s2012716; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start12717; GFX7-NEXT: ; =>This Inner Loop Header: Depth=112718; GFX7-NEXT: s_waitcnt vmcnt(0)12719; GFX7-NEXT: v_mov_b32_e32 v5, v012720; GFX7-NEXT: v_add_f32_e32 v4, v5, v212721; GFX7-NEXT: v_mov_b32_e32 v0, v412722; GFX7-NEXT: v_mov_b32_e32 v1, v512723; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc12724; GFX7-NEXT: s_waitcnt vmcnt(0)12725; GFX7-NEXT: buffer_wbinvl112726; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v512727; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]12728; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]12729; GFX7-NEXT: s_cbranch_execnz .LBB34_112730; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end12731; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]12732; GFX7-NEXT: s_setpc_b64 s[30:31]12733;12734; GFX6-LABEL: buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_grained_memory:12735; GFX6: ; %bb.0:12736; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)12737; GFX6-NEXT: v_mov_b32_e32 v2, v012738; GFX6-NEXT: v_mov_b32_e32 v0, s2012739; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:102412740; GFX6-NEXT: s_add_i32 s6, s20, 0x40012741; GFX6-NEXT: s_mov_b64 s[4:5], 012742; GFX6-NEXT: v_mov_b32_e32 v3, s612743; GFX6-NEXT: .LBB34_1: ; %atomicrmw.start12744; GFX6-NEXT: ; =>This Inner Loop Header: Depth=112745; GFX6-NEXT: s_waitcnt vmcnt(0)12746; GFX6-NEXT: v_mov_b32_e32 v5, v012747; GFX6-NEXT: v_add_f32_e32 v4, v5, v212748; GFX6-NEXT: s_waitcnt expcnt(0)12749; GFX6-NEXT: v_mov_b32_e32 v0, v412750; GFX6-NEXT: v_mov_b32_e32 v1, v512751; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc12752; GFX6-NEXT: s_waitcnt vmcnt(0)12753; GFX6-NEXT: buffer_wbinvl112754; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v512755; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]12756; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]12757; GFX6-NEXT: s_cbranch_execnz .LBB34_112758; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end12759; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]12760; GFX6-NEXT: s_waitcnt expcnt(0)12761; GFX6-NEXT: s_setpc_b64 s[30:31]12762 %gep = getelementptr float, ptr addrspace(7) %ptr, i32 25612763 %result = atomicrmw fadd ptr addrspace(7) %gep, float %val seq_cst, !amdgpu.no.fine.grained.memory !012764 ret float %result12765}12766 12767attributes #0 = { nounwind }12768 12769!0 = !{}12770