10124 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s3; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s4; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s5; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s6; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s7; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s8; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s9; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s10; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s11; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=hawaii < %s | FileCheck -check-prefix=GFX7 %s12; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck -check-prefix=GFX6 %s13 14; --------------------------------------------------------------------15; float16; --------------------------------------------------------------------17 18define float @local_atomic_fadd_ret_f32(ptr addrspace(3) %ptr) nounwind {19; GFX12-LABEL: local_atomic_fadd_ret_f32:20; GFX12: ; %bb.0:21; GFX12-NEXT: s_wait_loadcnt_dscnt 0x022; GFX12-NEXT: s_wait_expcnt 0x023; GFX12-NEXT: s_wait_samplecnt 0x024; GFX12-NEXT: s_wait_bvhcnt 0x025; GFX12-NEXT: s_wait_kmcnt 0x026; GFX12-NEXT: v_mov_b32_e32 v1, 4.027; GFX12-NEXT: s_wait_storecnt 0x028; GFX12-NEXT: ds_add_rtn_f32 v0, v0, v129; GFX12-NEXT: s_wait_dscnt 0x030; GFX12-NEXT: global_inv scope:SCOPE_SE31; GFX12-NEXT: s_wait_loadcnt 0x032; GFX12-NEXT: s_setpc_b64 s[30:31]33;34; GFX942-LABEL: local_atomic_fadd_ret_f32:35; GFX942: ; %bb.0:36; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)37; GFX942-NEXT: v_mov_b32_e32 v1, 4.038; GFX942-NEXT: ds_add_rtn_f32 v0, v0, v139; GFX942-NEXT: s_waitcnt lgkmcnt(0)40; GFX942-NEXT: s_setpc_b64 s[30:31]41;42; GFX11-LABEL: local_atomic_fadd_ret_f32:43; GFX11: ; %bb.0:44; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)45; GFX11-NEXT: v_mov_b32_e32 v1, 4.046; GFX11-NEXT: s_waitcnt_vscnt null, 0x047; GFX11-NEXT: ds_add_rtn_f32 v0, v0, v148; GFX11-NEXT: s_waitcnt lgkmcnt(0)49; GFX11-NEXT: buffer_gl0_inv50; GFX11-NEXT: s_setpc_b64 s[30:31]51;52; GFX10-LABEL: local_atomic_fadd_ret_f32:53; GFX10: ; %bb.0:54; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)55; GFX10-NEXT: v_mov_b32_e32 v1, 4.056; GFX10-NEXT: s_waitcnt_vscnt null, 0x057; GFX10-NEXT: ds_add_rtn_f32 v0, v0, v158; GFX10-NEXT: s_waitcnt lgkmcnt(0)59; GFX10-NEXT: buffer_gl0_inv60; GFX10-NEXT: s_setpc_b64 s[30:31]61;62; GFX90A-LABEL: local_atomic_fadd_ret_f32:63; GFX90A: ; %bb.0:64; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)65; GFX90A-NEXT: v_mov_b32_e32 v1, 4.066; GFX90A-NEXT: ds_add_rtn_f32 v0, v0, v167; GFX90A-NEXT: s_waitcnt lgkmcnt(0)68; GFX90A-NEXT: s_setpc_b64 s[30:31]69;70; GFX908-LABEL: local_atomic_fadd_ret_f32:71; GFX908: ; %bb.0:72; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)73; GFX908-NEXT: v_mov_b32_e32 v1, 4.074; GFX908-NEXT: ds_add_rtn_f32 v0, v0, v175; GFX908-NEXT: s_waitcnt lgkmcnt(0)76; GFX908-NEXT: s_setpc_b64 s[30:31]77;78; GFX8-LABEL: local_atomic_fadd_ret_f32:79; GFX8: ; %bb.0:80; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)81; GFX8-NEXT: v_mov_b32_e32 v1, 4.082; GFX8-NEXT: s_mov_b32 m0, -183; GFX8-NEXT: ds_add_rtn_f32 v0, v0, v184; GFX8-NEXT: s_waitcnt lgkmcnt(0)85; GFX8-NEXT: s_setpc_b64 s[30:31]86;87; GFX7-LABEL: local_atomic_fadd_ret_f32:88; GFX7: ; %bb.0:89; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)90; GFX7-NEXT: s_mov_b32 m0, -191; GFX7-NEXT: ds_read_b32 v1, v092; GFX7-NEXT: s_mov_b64 s[4:5], 093; GFX7-NEXT: .LBB0_1: ; %atomicrmw.start94; GFX7-NEXT: ; =>This Inner Loop Header: Depth=195; GFX7-NEXT: s_waitcnt lgkmcnt(0)96; GFX7-NEXT: v_mov_b32_e32 v2, v197; GFX7-NEXT: v_add_f32_e32 v1, 4.0, v298; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v199; GFX7-NEXT: s_waitcnt lgkmcnt(0)100; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2101; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]102; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]103; GFX7-NEXT: s_cbranch_execnz .LBB0_1104; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end105; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]106; GFX7-NEXT: v_mov_b32_e32 v0, v1107; GFX7-NEXT: s_setpc_b64 s[30:31]108;109; GFX6-LABEL: local_atomic_fadd_ret_f32:110; GFX6: ; %bb.0:111; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)112; GFX6-NEXT: s_mov_b32 m0, -1113; GFX6-NEXT: ds_read_b32 v1, v0114; GFX6-NEXT: s_mov_b64 s[4:5], 0115; GFX6-NEXT: .LBB0_1: ; %atomicrmw.start116; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1117; GFX6-NEXT: s_waitcnt lgkmcnt(0)118; GFX6-NEXT: v_mov_b32_e32 v2, v1119; GFX6-NEXT: v_add_f32_e32 v1, 4.0, v2120; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1121; GFX6-NEXT: s_waitcnt lgkmcnt(0)122; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2123; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]124; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]125; GFX6-NEXT: s_cbranch_execnz .LBB0_1126; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end127; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]128; GFX6-NEXT: v_mov_b32_e32 v0, v1129; GFX6-NEXT: s_setpc_b64 s[30:31]130 %result = atomicrmw fadd ptr addrspace(3) %ptr, float 4.0 seq_cst131 ret float %result132}133 134define float @local_atomic_fadd_ret_f32__offset(ptr addrspace(3) %ptr) nounwind {135; GFX12-LABEL: local_atomic_fadd_ret_f32__offset:136; GFX12: ; %bb.0:137; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0138; GFX12-NEXT: s_wait_expcnt 0x0139; GFX12-NEXT: s_wait_samplecnt 0x0140; GFX12-NEXT: s_wait_bvhcnt 0x0141; GFX12-NEXT: s_wait_kmcnt 0x0142; GFX12-NEXT: v_mov_b32_e32 v1, 4.0143; GFX12-NEXT: s_wait_storecnt 0x0144; GFX12-NEXT: ds_add_rtn_f32 v0, v0, v1 offset:65532145; GFX12-NEXT: s_wait_dscnt 0x0146; GFX12-NEXT: global_inv scope:SCOPE_SE147; GFX12-NEXT: s_wait_loadcnt 0x0148; GFX12-NEXT: s_setpc_b64 s[30:31]149;150; GFX942-LABEL: local_atomic_fadd_ret_f32__offset:151; GFX942: ; %bb.0:152; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)153; GFX942-NEXT: v_mov_b32_e32 v1, 4.0154; GFX942-NEXT: ds_add_rtn_f32 v0, v0, v1 offset:65532155; GFX942-NEXT: s_waitcnt lgkmcnt(0)156; GFX942-NEXT: s_setpc_b64 s[30:31]157;158; GFX11-LABEL: local_atomic_fadd_ret_f32__offset:159; GFX11: ; %bb.0:160; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)161; GFX11-NEXT: v_mov_b32_e32 v1, 4.0162; GFX11-NEXT: s_waitcnt_vscnt null, 0x0163; GFX11-NEXT: ds_add_rtn_f32 v0, v0, v1 offset:65532164; GFX11-NEXT: s_waitcnt lgkmcnt(0)165; GFX11-NEXT: buffer_gl0_inv166; GFX11-NEXT: s_setpc_b64 s[30:31]167;168; GFX10-LABEL: local_atomic_fadd_ret_f32__offset:169; GFX10: ; %bb.0:170; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)171; GFX10-NEXT: v_mov_b32_e32 v1, 4.0172; GFX10-NEXT: s_waitcnt_vscnt null, 0x0173; GFX10-NEXT: ds_add_rtn_f32 v0, v0, v1 offset:65532174; GFX10-NEXT: s_waitcnt lgkmcnt(0)175; GFX10-NEXT: buffer_gl0_inv176; GFX10-NEXT: s_setpc_b64 s[30:31]177;178; GFX90A-LABEL: local_atomic_fadd_ret_f32__offset:179; GFX90A: ; %bb.0:180; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)181; GFX90A-NEXT: v_mov_b32_e32 v1, 4.0182; GFX90A-NEXT: ds_add_rtn_f32 v0, v0, v1 offset:65532183; GFX90A-NEXT: s_waitcnt lgkmcnt(0)184; GFX90A-NEXT: s_setpc_b64 s[30:31]185;186; GFX908-LABEL: local_atomic_fadd_ret_f32__offset:187; GFX908: ; %bb.0:188; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)189; GFX908-NEXT: v_mov_b32_e32 v1, 4.0190; GFX908-NEXT: ds_add_rtn_f32 v0, v0, v1 offset:65532191; GFX908-NEXT: s_waitcnt lgkmcnt(0)192; GFX908-NEXT: s_setpc_b64 s[30:31]193;194; GFX8-LABEL: local_atomic_fadd_ret_f32__offset:195; GFX8: ; %bb.0:196; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)197; GFX8-NEXT: v_mov_b32_e32 v1, 4.0198; GFX8-NEXT: s_mov_b32 m0, -1199; GFX8-NEXT: ds_add_rtn_f32 v0, v0, v1 offset:65532200; GFX8-NEXT: s_waitcnt lgkmcnt(0)201; GFX8-NEXT: s_setpc_b64 s[30:31]202;203; GFX7-LABEL: local_atomic_fadd_ret_f32__offset:204; GFX7: ; %bb.0:205; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)206; GFX7-NEXT: s_mov_b32 m0, -1207; GFX7-NEXT: ds_read_b32 v1, v0 offset:65532208; GFX7-NEXT: s_mov_b64 s[4:5], 0209; GFX7-NEXT: .LBB1_1: ; %atomicrmw.start210; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1211; GFX7-NEXT: s_waitcnt lgkmcnt(0)212; GFX7-NEXT: v_mov_b32_e32 v2, v1213; GFX7-NEXT: v_add_f32_e32 v1, 4.0, v2214; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:65532215; GFX7-NEXT: s_waitcnt lgkmcnt(0)216; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2217; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]218; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]219; GFX7-NEXT: s_cbranch_execnz .LBB1_1220; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end221; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]222; GFX7-NEXT: v_mov_b32_e32 v0, v1223; GFX7-NEXT: s_setpc_b64 s[30:31]224;225; GFX6-LABEL: local_atomic_fadd_ret_f32__offset:226; GFX6: ; %bb.0:227; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)228; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffc, v0229; GFX6-NEXT: s_mov_b32 m0, -1230; GFX6-NEXT: ds_read_b32 v0, v1231; GFX6-NEXT: s_mov_b64 s[4:5], 0232; GFX6-NEXT: .LBB1_1: ; %atomicrmw.start233; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1234; GFX6-NEXT: s_waitcnt lgkmcnt(0)235; GFX6-NEXT: v_mov_b32_e32 v2, v0236; GFX6-NEXT: v_add_f32_e32 v0, 4.0, v2237; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0238; GFX6-NEXT: s_waitcnt lgkmcnt(0)239; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2240; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]241; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]242; GFX6-NEXT: s_cbranch_execnz .LBB1_1243; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end244; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]245; GFX6-NEXT: s_setpc_b64 s[30:31]246 %gep = getelementptr float, ptr addrspace(3) %ptr, i32 16383247 %result = atomicrmw fadd ptr addrspace(3) %gep, float 4.0 seq_cst248 ret float %result249}250 251define void @local_atomic_fadd_noret_f32(ptr addrspace(3) %ptr) nounwind {252; GFX12-LABEL: local_atomic_fadd_noret_f32:253; GFX12: ; %bb.0:254; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0255; GFX12-NEXT: s_wait_expcnt 0x0256; GFX12-NEXT: s_wait_samplecnt 0x0257; GFX12-NEXT: s_wait_bvhcnt 0x0258; GFX12-NEXT: s_wait_kmcnt 0x0259; GFX12-NEXT: v_mov_b32_e32 v1, 4.0260; GFX12-NEXT: s_wait_storecnt 0x0261; GFX12-NEXT: ds_add_f32 v0, v1262; GFX12-NEXT: s_wait_dscnt 0x0263; GFX12-NEXT: global_inv scope:SCOPE_SE264; GFX12-NEXT: s_wait_loadcnt 0x0265; GFX12-NEXT: s_setpc_b64 s[30:31]266;267; GFX942-LABEL: local_atomic_fadd_noret_f32:268; GFX942: ; %bb.0:269; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)270; GFX942-NEXT: v_mov_b32_e32 v1, 4.0271; GFX942-NEXT: ds_add_f32 v0, v1272; GFX942-NEXT: s_waitcnt lgkmcnt(0)273; GFX942-NEXT: s_setpc_b64 s[30:31]274;275; GFX11-LABEL: local_atomic_fadd_noret_f32:276; GFX11: ; %bb.0:277; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)278; GFX11-NEXT: v_mov_b32_e32 v1, 4.0279; GFX11-NEXT: s_waitcnt_vscnt null, 0x0280; GFX11-NEXT: ds_add_f32 v0, v1281; GFX11-NEXT: s_waitcnt lgkmcnt(0)282; GFX11-NEXT: buffer_gl0_inv283; GFX11-NEXT: s_setpc_b64 s[30:31]284;285; GFX10-LABEL: local_atomic_fadd_noret_f32:286; GFX10: ; %bb.0:287; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)288; GFX10-NEXT: v_mov_b32_e32 v1, 4.0289; GFX10-NEXT: s_waitcnt_vscnt null, 0x0290; GFX10-NEXT: ds_add_f32 v0, v1291; GFX10-NEXT: s_waitcnt lgkmcnt(0)292; GFX10-NEXT: buffer_gl0_inv293; GFX10-NEXT: s_setpc_b64 s[30:31]294;295; GFX90A-LABEL: local_atomic_fadd_noret_f32:296; GFX90A: ; %bb.0:297; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)298; GFX90A-NEXT: v_mov_b32_e32 v1, 4.0299; GFX90A-NEXT: ds_add_f32 v0, v1300; GFX90A-NEXT: s_waitcnt lgkmcnt(0)301; GFX90A-NEXT: s_setpc_b64 s[30:31]302;303; GFX908-LABEL: local_atomic_fadd_noret_f32:304; GFX908: ; %bb.0:305; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)306; GFX908-NEXT: v_mov_b32_e32 v1, 4.0307; GFX908-NEXT: ds_add_f32 v0, v1308; GFX908-NEXT: s_waitcnt lgkmcnt(0)309; GFX908-NEXT: s_setpc_b64 s[30:31]310;311; GFX8-LABEL: local_atomic_fadd_noret_f32:312; GFX8: ; %bb.0:313; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)314; GFX8-NEXT: v_mov_b32_e32 v1, 4.0315; GFX8-NEXT: s_mov_b32 m0, -1316; GFX8-NEXT: ds_add_f32 v0, v1317; GFX8-NEXT: s_waitcnt lgkmcnt(0)318; GFX8-NEXT: s_setpc_b64 s[30:31]319;320; GFX7-LABEL: local_atomic_fadd_noret_f32:321; GFX7: ; %bb.0:322; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)323; GFX7-NEXT: s_mov_b32 m0, -1324; GFX7-NEXT: ds_read_b32 v1, v0325; GFX7-NEXT: s_mov_b64 s[4:5], 0326; GFX7-NEXT: .LBB2_1: ; %atomicrmw.start327; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1328; GFX7-NEXT: s_waitcnt lgkmcnt(0)329; GFX7-NEXT: v_add_f32_e32 v2, 4.0, v1330; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2331; GFX7-NEXT: s_waitcnt lgkmcnt(0)332; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1333; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]334; GFX7-NEXT: v_mov_b32_e32 v1, v2335; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]336; GFX7-NEXT: s_cbranch_execnz .LBB2_1337; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end338; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]339; GFX7-NEXT: s_setpc_b64 s[30:31]340;341; GFX6-LABEL: local_atomic_fadd_noret_f32:342; GFX6: ; %bb.0:343; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)344; GFX6-NEXT: s_mov_b32 m0, -1345; GFX6-NEXT: ds_read_b32 v1, v0346; GFX6-NEXT: s_mov_b64 s[4:5], 0347; GFX6-NEXT: .LBB2_1: ; %atomicrmw.start348; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1349; GFX6-NEXT: s_waitcnt lgkmcnt(0)350; GFX6-NEXT: v_add_f32_e32 v2, 4.0, v1351; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2352; GFX6-NEXT: s_waitcnt lgkmcnt(0)353; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1354; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]355; GFX6-NEXT: v_mov_b32_e32 v1, v2356; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]357; GFX6-NEXT: s_cbranch_execnz .LBB2_1358; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end359; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]360; GFX6-NEXT: s_setpc_b64 s[30:31]361 %result = atomicrmw fadd ptr addrspace(3) %ptr, float 4.0 seq_cst362 ret void363}364 365define void @local_atomic_fadd_noret_f32__offset(ptr addrspace(3) %ptr) nounwind {366; GFX12-LABEL: local_atomic_fadd_noret_f32__offset:367; GFX12: ; %bb.0:368; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0369; GFX12-NEXT: s_wait_expcnt 0x0370; GFX12-NEXT: s_wait_samplecnt 0x0371; GFX12-NEXT: s_wait_bvhcnt 0x0372; GFX12-NEXT: s_wait_kmcnt 0x0373; GFX12-NEXT: v_mov_b32_e32 v1, 4.0374; GFX12-NEXT: s_wait_storecnt 0x0375; GFX12-NEXT: ds_add_f32 v0, v1 offset:65532376; GFX12-NEXT: s_wait_dscnt 0x0377; GFX12-NEXT: global_inv scope:SCOPE_SE378; GFX12-NEXT: s_wait_loadcnt 0x0379; GFX12-NEXT: s_setpc_b64 s[30:31]380;381; GFX942-LABEL: local_atomic_fadd_noret_f32__offset:382; GFX942: ; %bb.0:383; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)384; GFX942-NEXT: v_mov_b32_e32 v1, 4.0385; GFX942-NEXT: ds_add_f32 v0, v1 offset:65532386; GFX942-NEXT: s_waitcnt lgkmcnt(0)387; GFX942-NEXT: s_setpc_b64 s[30:31]388;389; GFX11-LABEL: local_atomic_fadd_noret_f32__offset:390; GFX11: ; %bb.0:391; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)392; GFX11-NEXT: v_mov_b32_e32 v1, 4.0393; GFX11-NEXT: s_waitcnt_vscnt null, 0x0394; GFX11-NEXT: ds_add_f32 v0, v1 offset:65532395; GFX11-NEXT: s_waitcnt lgkmcnt(0)396; GFX11-NEXT: buffer_gl0_inv397; GFX11-NEXT: s_setpc_b64 s[30:31]398;399; GFX10-LABEL: local_atomic_fadd_noret_f32__offset:400; GFX10: ; %bb.0:401; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)402; GFX10-NEXT: v_mov_b32_e32 v1, 4.0403; GFX10-NEXT: s_waitcnt_vscnt null, 0x0404; GFX10-NEXT: ds_add_f32 v0, v1 offset:65532405; GFX10-NEXT: s_waitcnt lgkmcnt(0)406; GFX10-NEXT: buffer_gl0_inv407; GFX10-NEXT: s_setpc_b64 s[30:31]408;409; GFX90A-LABEL: local_atomic_fadd_noret_f32__offset:410; GFX90A: ; %bb.0:411; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)412; GFX90A-NEXT: v_mov_b32_e32 v1, 4.0413; GFX90A-NEXT: ds_add_f32 v0, v1 offset:65532414; GFX90A-NEXT: s_waitcnt lgkmcnt(0)415; GFX90A-NEXT: s_setpc_b64 s[30:31]416;417; GFX908-LABEL: local_atomic_fadd_noret_f32__offset:418; GFX908: ; %bb.0:419; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)420; GFX908-NEXT: v_mov_b32_e32 v1, 4.0421; GFX908-NEXT: ds_add_f32 v0, v1 offset:65532422; GFX908-NEXT: s_waitcnt lgkmcnt(0)423; GFX908-NEXT: s_setpc_b64 s[30:31]424;425; GFX8-LABEL: local_atomic_fadd_noret_f32__offset:426; GFX8: ; %bb.0:427; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)428; GFX8-NEXT: v_mov_b32_e32 v1, 4.0429; GFX8-NEXT: s_mov_b32 m0, -1430; GFX8-NEXT: ds_add_f32 v0, v1 offset:65532431; GFX8-NEXT: s_waitcnt lgkmcnt(0)432; GFX8-NEXT: s_setpc_b64 s[30:31]433;434; GFX7-LABEL: local_atomic_fadd_noret_f32__offset:435; GFX7: ; %bb.0:436; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)437; GFX7-NEXT: s_mov_b32 m0, -1438; GFX7-NEXT: ds_read_b32 v1, v0 offset:65532439; GFX7-NEXT: s_mov_b64 s[4:5], 0440; GFX7-NEXT: .LBB3_1: ; %atomicrmw.start441; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1442; GFX7-NEXT: s_waitcnt lgkmcnt(0)443; GFX7-NEXT: v_add_f32_e32 v2, 4.0, v1444; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:65532445; GFX7-NEXT: s_waitcnt lgkmcnt(0)446; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1447; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]448; GFX7-NEXT: v_mov_b32_e32 v1, v2449; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]450; GFX7-NEXT: s_cbranch_execnz .LBB3_1451; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end452; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]453; GFX7-NEXT: s_setpc_b64 s[30:31]454;455; GFX6-LABEL: local_atomic_fadd_noret_f32__offset:456; GFX6: ; %bb.0:457; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)458; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0459; GFX6-NEXT: s_mov_b32 m0, -1460; GFX6-NEXT: ds_read_b32 v1, v0461; GFX6-NEXT: s_mov_b64 s[4:5], 0462; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start463; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1464; GFX6-NEXT: s_waitcnt lgkmcnt(0)465; GFX6-NEXT: v_add_f32_e32 v2, 4.0, v1466; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2467; GFX6-NEXT: s_waitcnt lgkmcnt(0)468; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1469; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]470; GFX6-NEXT: v_mov_b32_e32 v1, v2471; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]472; GFX6-NEXT: s_cbranch_execnz .LBB3_1473; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end474; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]475; GFX6-NEXT: s_setpc_b64 s[30:31]476 %gep = getelementptr float, ptr addrspace(3) %ptr, i32 16383477 %unused = atomicrmw fadd ptr addrspace(3) %gep, float 4.0 seq_cst478 ret void479}480 481; --------------------------------------------------------------------482; double483; --------------------------------------------------------------------484 485define double @local_atomic_fadd_ret_f64(ptr addrspace(3) %ptr) nounwind {486; GFX12-LABEL: local_atomic_fadd_ret_f64:487; GFX12: ; %bb.0:488; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0489; GFX12-NEXT: s_wait_expcnt 0x0490; GFX12-NEXT: s_wait_samplecnt 0x0491; GFX12-NEXT: s_wait_bvhcnt 0x0492; GFX12-NEXT: s_wait_kmcnt 0x0493; GFX12-NEXT: v_mov_b32_e32 v2, v0494; GFX12-NEXT: ds_load_b64 v[0:1], v0495; GFX12-NEXT: s_mov_b32 s0, 0496; GFX12-NEXT: .LBB4_1: ; %atomicrmw.start497; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1498; GFX12-NEXT: s_wait_dscnt 0x0499; GFX12-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0500; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)501; GFX12-NEXT: v_add_f64_e32 v[0:1], 4.0, v[3:4]502; GFX12-NEXT: s_wait_storecnt 0x0503; GFX12-NEXT: ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[3:4]504; GFX12-NEXT: s_wait_dscnt 0x0505; GFX12-NEXT: global_inv scope:SCOPE_SE506; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]507; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)508; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0509; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)510; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0511; GFX12-NEXT: s_cbranch_execnz .LBB4_1512; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end513; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0514; GFX12-NEXT: s_wait_loadcnt 0x0515; GFX12-NEXT: s_setpc_b64 s[30:31]516;517; GFX942-LABEL: local_atomic_fadd_ret_f64:518; GFX942: ; %bb.0:519; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)520; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.0521; GFX942-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]522; GFX942-NEXT: s_waitcnt lgkmcnt(0)523; GFX942-NEXT: s_setpc_b64 s[30:31]524;525; GFX11-LABEL: local_atomic_fadd_ret_f64:526; GFX11: ; %bb.0:527; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)528; GFX11-NEXT: v_mov_b32_e32 v2, v0529; GFX11-NEXT: ds_load_b64 v[0:1], v0530; GFX11-NEXT: s_mov_b32 s0, 0531; GFX11-NEXT: .LBB4_1: ; %atomicrmw.start532; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1533; GFX11-NEXT: s_waitcnt lgkmcnt(0)534; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0535; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)536; GFX11-NEXT: v_add_f64 v[0:1], v[3:4], 4.0537; GFX11-NEXT: s_waitcnt_vscnt null, 0x0538; GFX11-NEXT: ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[3:4]539; GFX11-NEXT: s_waitcnt lgkmcnt(0)540; GFX11-NEXT: buffer_gl0_inv541; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]542; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0543; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)544; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0545; GFX11-NEXT: s_cbranch_execnz .LBB4_1546; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end547; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0548; GFX11-NEXT: s_setpc_b64 s[30:31]549;550; GFX10-LABEL: local_atomic_fadd_ret_f64:551; GFX10: ; %bb.0:552; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)553; GFX10-NEXT: v_mov_b32_e32 v2, v0554; GFX10-NEXT: ds_read_b64 v[0:1], v0555; GFX10-NEXT: s_mov_b32 s4, 0556; GFX10-NEXT: .LBB4_1: ; %atomicrmw.start557; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1558; GFX10-NEXT: s_waitcnt lgkmcnt(0)559; GFX10-NEXT: v_mov_b32_e32 v4, v1560; GFX10-NEXT: v_mov_b32_e32 v3, v0561; GFX10-NEXT: v_add_f64 v[0:1], v[3:4], 4.0562; GFX10-NEXT: s_waitcnt_vscnt null, 0x0563; GFX10-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]564; GFX10-NEXT: s_waitcnt lgkmcnt(0)565; GFX10-NEXT: buffer_gl0_inv566; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]567; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4568; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4569; GFX10-NEXT: s_cbranch_execnz .LBB4_1570; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end571; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4572; GFX10-NEXT: s_setpc_b64 s[30:31]573;574; GFX90A-LABEL: local_atomic_fadd_ret_f64:575; GFX90A: ; %bb.0:576; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)577; GFX90A-NEXT: v_mov_b32_e32 v2, 0578; GFX90A-NEXT: v_mov_b32_e32 v3, 0x40100000579; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3]580; GFX90A-NEXT: s_waitcnt lgkmcnt(0)581; GFX90A-NEXT: s_setpc_b64 s[30:31]582;583; GFX908-LABEL: local_atomic_fadd_ret_f64:584; GFX908: ; %bb.0:585; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)586; GFX908-NEXT: v_mov_b32_e32 v2, v0587; GFX908-NEXT: ds_read_b64 v[0:1], v0588; GFX908-NEXT: s_mov_b64 s[4:5], 0589; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start590; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1591; GFX908-NEXT: s_waitcnt lgkmcnt(0)592; GFX908-NEXT: v_mov_b32_e32 v4, v1593; GFX908-NEXT: v_mov_b32_e32 v3, v0594; GFX908-NEXT: v_add_f64 v[0:1], v[3:4], 4.0595; GFX908-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]596; GFX908-NEXT: s_waitcnt lgkmcnt(0)597; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]598; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]599; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]600; GFX908-NEXT: s_cbranch_execnz .LBB4_1601; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end602; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]603; GFX908-NEXT: s_setpc_b64 s[30:31]604;605; GFX8-LABEL: local_atomic_fadd_ret_f64:606; GFX8: ; %bb.0:607; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)608; GFX8-NEXT: s_mov_b32 m0, -1609; GFX8-NEXT: v_mov_b32_e32 v2, v0610; GFX8-NEXT: ds_read_b64 v[0:1], v0611; GFX8-NEXT: s_mov_b64 s[4:5], 0612; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start613; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1614; GFX8-NEXT: s_waitcnt lgkmcnt(0)615; GFX8-NEXT: v_mov_b32_e32 v4, v1616; GFX8-NEXT: v_mov_b32_e32 v3, v0617; GFX8-NEXT: v_add_f64 v[0:1], v[3:4], 4.0618; GFX8-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]619; GFX8-NEXT: s_waitcnt lgkmcnt(0)620; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]621; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]622; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]623; GFX8-NEXT: s_cbranch_execnz .LBB4_1624; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end625; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]626; GFX8-NEXT: s_setpc_b64 s[30:31]627;628; GFX7-LABEL: local_atomic_fadd_ret_f64:629; GFX7: ; %bb.0:630; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)631; GFX7-NEXT: s_mov_b32 m0, -1632; GFX7-NEXT: v_mov_b32_e32 v2, v0633; GFX7-NEXT: ds_read_b64 v[0:1], v0634; GFX7-NEXT: s_mov_b64 s[4:5], 0635; GFX7-NEXT: .LBB4_1: ; %atomicrmw.start636; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1637; GFX7-NEXT: s_waitcnt lgkmcnt(0)638; GFX7-NEXT: v_mov_b32_e32 v4, v1639; GFX7-NEXT: v_mov_b32_e32 v3, v0640; GFX7-NEXT: v_add_f64 v[0:1], v[3:4], 4.0641; GFX7-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]642; GFX7-NEXT: s_waitcnt lgkmcnt(0)643; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]644; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]645; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]646; GFX7-NEXT: s_cbranch_execnz .LBB4_1647; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end648; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]649; GFX7-NEXT: s_setpc_b64 s[30:31]650;651; GFX6-LABEL: local_atomic_fadd_ret_f64:652; GFX6: ; %bb.0:653; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)654; GFX6-NEXT: s_mov_b32 m0, -1655; GFX6-NEXT: v_mov_b32_e32 v2, v0656; GFX6-NEXT: ds_read_b64 v[0:1], v0657; GFX6-NEXT: s_mov_b64 s[4:5], 0658; GFX6-NEXT: .LBB4_1: ; %atomicrmw.start659; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1660; GFX6-NEXT: s_waitcnt lgkmcnt(0)661; GFX6-NEXT: v_mov_b32_e32 v4, v1662; GFX6-NEXT: v_mov_b32_e32 v3, v0663; GFX6-NEXT: v_add_f64 v[0:1], v[3:4], 4.0664; GFX6-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]665; GFX6-NEXT: s_waitcnt lgkmcnt(0)666; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]667; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]668; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]669; GFX6-NEXT: s_cbranch_execnz .LBB4_1670; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end671; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]672; GFX6-NEXT: s_setpc_b64 s[30:31]673 %result = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst674 ret double %result675}676 677define double @local_atomic_fadd_ret_f64__offset(ptr addrspace(3) %ptr) nounwind {678; GFX12-LABEL: local_atomic_fadd_ret_f64__offset:679; GFX12: ; %bb.0:680; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0681; GFX12-NEXT: s_wait_expcnt 0x0682; GFX12-NEXT: s_wait_samplecnt 0x0683; GFX12-NEXT: s_wait_bvhcnt 0x0684; GFX12-NEXT: s_wait_kmcnt 0x0685; GFX12-NEXT: v_mov_b32_e32 v2, v0686; GFX12-NEXT: ds_load_b64 v[0:1], v0 offset:65528687; GFX12-NEXT: s_mov_b32 s0, 0688; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start689; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1690; GFX12-NEXT: s_wait_dscnt 0x0691; GFX12-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0692; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)693; GFX12-NEXT: v_add_f64_e32 v[0:1], 4.0, v[3:4]694; GFX12-NEXT: s_wait_storecnt 0x0695; GFX12-NEXT: ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[3:4] offset:65528696; GFX12-NEXT: s_wait_dscnt 0x0697; GFX12-NEXT: global_inv scope:SCOPE_SE698; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]699; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)700; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0701; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)702; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0703; GFX12-NEXT: s_cbranch_execnz .LBB5_1704; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end705; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0706; GFX12-NEXT: s_wait_loadcnt 0x0707; GFX12-NEXT: s_setpc_b64 s[30:31]708;709; GFX942-LABEL: local_atomic_fadd_ret_f64__offset:710; GFX942: ; %bb.0:711; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)712; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.0713; GFX942-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3] offset:65528714; GFX942-NEXT: s_waitcnt lgkmcnt(0)715; GFX942-NEXT: s_setpc_b64 s[30:31]716;717; GFX11-LABEL: local_atomic_fadd_ret_f64__offset:718; GFX11: ; %bb.0:719; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)720; GFX11-NEXT: v_mov_b32_e32 v2, v0721; GFX11-NEXT: ds_load_b64 v[0:1], v0 offset:65528722; GFX11-NEXT: s_mov_b32 s0, 0723; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start724; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1725; GFX11-NEXT: s_waitcnt lgkmcnt(0)726; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0727; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)728; GFX11-NEXT: v_add_f64 v[0:1], v[3:4], 4.0729; GFX11-NEXT: s_waitcnt_vscnt null, 0x0730; GFX11-NEXT: ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[3:4] offset:65528731; GFX11-NEXT: s_waitcnt lgkmcnt(0)732; GFX11-NEXT: buffer_gl0_inv733; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]734; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0735; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)736; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0737; GFX11-NEXT: s_cbranch_execnz .LBB5_1738; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end739; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0740; GFX11-NEXT: s_setpc_b64 s[30:31]741;742; GFX10-LABEL: local_atomic_fadd_ret_f64__offset:743; GFX10: ; %bb.0:744; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)745; GFX10-NEXT: v_mov_b32_e32 v2, v0746; GFX10-NEXT: ds_read_b64 v[0:1], v0 offset:65528747; GFX10-NEXT: s_mov_b32 s4, 0748; GFX10-NEXT: .LBB5_1: ; %atomicrmw.start749; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1750; GFX10-NEXT: s_waitcnt lgkmcnt(0)751; GFX10-NEXT: v_mov_b32_e32 v4, v1752; GFX10-NEXT: v_mov_b32_e32 v3, v0753; GFX10-NEXT: v_add_f64 v[0:1], v[3:4], 4.0754; GFX10-NEXT: s_waitcnt_vscnt null, 0x0755; GFX10-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1] offset:65528756; GFX10-NEXT: s_waitcnt lgkmcnt(0)757; GFX10-NEXT: buffer_gl0_inv758; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[3:4]759; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4760; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4761; GFX10-NEXT: s_cbranch_execnz .LBB5_1762; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end763; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4764; GFX10-NEXT: s_setpc_b64 s[30:31]765;766; GFX90A-LABEL: local_atomic_fadd_ret_f64__offset:767; GFX90A: ; %bb.0:768; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)769; GFX90A-NEXT: v_mov_b32_e32 v2, 0770; GFX90A-NEXT: v_mov_b32_e32 v3, 0x40100000771; GFX90A-NEXT: ds_add_rtn_f64 v[0:1], v0, v[2:3] offset:65528772; GFX90A-NEXT: s_waitcnt lgkmcnt(0)773; GFX90A-NEXT: s_setpc_b64 s[30:31]774;775; GFX908-LABEL: local_atomic_fadd_ret_f64__offset:776; GFX908: ; %bb.0:777; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)778; GFX908-NEXT: v_mov_b32_e32 v2, v0779; GFX908-NEXT: ds_read_b64 v[0:1], v0 offset:65528780; GFX908-NEXT: s_mov_b64 s[4:5], 0781; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start782; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1783; GFX908-NEXT: s_waitcnt lgkmcnt(0)784; GFX908-NEXT: v_mov_b32_e32 v4, v1785; GFX908-NEXT: v_mov_b32_e32 v3, v0786; GFX908-NEXT: v_add_f64 v[0:1], v[3:4], 4.0787; GFX908-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1] offset:65528788; GFX908-NEXT: s_waitcnt lgkmcnt(0)789; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]790; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]791; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]792; GFX908-NEXT: s_cbranch_execnz .LBB5_1793; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end794; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]795; GFX908-NEXT: s_setpc_b64 s[30:31]796;797; GFX8-LABEL: local_atomic_fadd_ret_f64__offset:798; GFX8: ; %bb.0:799; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)800; GFX8-NEXT: s_mov_b32 m0, -1801; GFX8-NEXT: v_mov_b32_e32 v2, v0802; GFX8-NEXT: ds_read_b64 v[0:1], v0 offset:65528803; GFX8-NEXT: s_mov_b64 s[4:5], 0804; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start805; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1806; GFX8-NEXT: s_waitcnt lgkmcnt(0)807; GFX8-NEXT: v_mov_b32_e32 v4, v1808; GFX8-NEXT: v_mov_b32_e32 v3, v0809; GFX8-NEXT: v_add_f64 v[0:1], v[3:4], 4.0810; GFX8-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1] offset:65528811; GFX8-NEXT: s_waitcnt lgkmcnt(0)812; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]813; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]814; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]815; GFX8-NEXT: s_cbranch_execnz .LBB5_1816; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end817; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]818; GFX8-NEXT: s_setpc_b64 s[30:31]819;820; GFX7-LABEL: local_atomic_fadd_ret_f64__offset:821; GFX7: ; %bb.0:822; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)823; GFX7-NEXT: s_mov_b32 m0, -1824; GFX7-NEXT: v_mov_b32_e32 v2, v0825; GFX7-NEXT: ds_read_b64 v[0:1], v0 offset:65528826; GFX7-NEXT: s_mov_b64 s[4:5], 0827; GFX7-NEXT: .LBB5_1: ; %atomicrmw.start828; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1829; GFX7-NEXT: s_waitcnt lgkmcnt(0)830; GFX7-NEXT: v_mov_b32_e32 v4, v1831; GFX7-NEXT: v_mov_b32_e32 v3, v0832; GFX7-NEXT: v_add_f64 v[0:1], v[3:4], 4.0833; GFX7-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1] offset:65528834; GFX7-NEXT: s_waitcnt lgkmcnt(0)835; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]836; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]837; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]838; GFX7-NEXT: s_cbranch_execnz .LBB5_1839; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end840; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]841; GFX7-NEXT: s_setpc_b64 s[30:31]842;843; GFX6-LABEL: local_atomic_fadd_ret_f64__offset:844; GFX6: ; %bb.0:845; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)846; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfff8, v0847; GFX6-NEXT: s_mov_b32 m0, -1848; GFX6-NEXT: ds_read_b64 v[0:1], v2849; GFX6-NEXT: s_mov_b64 s[4:5], 0850; GFX6-NEXT: .LBB5_1: ; %atomicrmw.start851; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1852; GFX6-NEXT: s_waitcnt lgkmcnt(0)853; GFX6-NEXT: v_mov_b32_e32 v4, v1854; GFX6-NEXT: v_mov_b32_e32 v3, v0855; GFX6-NEXT: v_add_f64 v[0:1], v[3:4], 4.0856; GFX6-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]857; GFX6-NEXT: s_waitcnt lgkmcnt(0)858; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]859; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]860; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]861; GFX6-NEXT: s_cbranch_execnz .LBB5_1862; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end863; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]864; GFX6-NEXT: s_setpc_b64 s[30:31]865 %gep = getelementptr double, ptr addrspace(3) %ptr, i32 8191866 %result = atomicrmw fadd ptr addrspace(3) %gep, double 4.0 seq_cst867 ret double %result868}869 870define void @local_atomic_fadd_noret_f64(ptr addrspace(3) %ptr) nounwind {871; GFX12-LABEL: local_atomic_fadd_noret_f64:872; GFX12: ; %bb.0:873; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0874; GFX12-NEXT: s_wait_expcnt 0x0875; GFX12-NEXT: s_wait_samplecnt 0x0876; GFX12-NEXT: s_wait_bvhcnt 0x0877; GFX12-NEXT: s_wait_kmcnt 0x0878; GFX12-NEXT: ds_load_b64 v[1:2], v0879; GFX12-NEXT: s_mov_b32 s0, 0880; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start881; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1882; GFX12-NEXT: s_wait_dscnt 0x0883; GFX12-NEXT: v_add_f64_e32 v[3:4], 4.0, v[1:2]884; GFX12-NEXT: s_wait_storecnt 0x0885; GFX12-NEXT: ds_cmpstore_rtn_b64 v[3:4], v0, v[3:4], v[1:2]886; GFX12-NEXT: s_wait_dscnt 0x0887; GFX12-NEXT: global_inv scope:SCOPE_SE888; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]889; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4890; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)891; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0892; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)893; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0894; GFX12-NEXT: s_cbranch_execnz .LBB6_1895; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end896; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0897; GFX12-NEXT: s_wait_loadcnt 0x0898; GFX12-NEXT: s_setpc_b64 s[30:31]899;900; GFX942-LABEL: local_atomic_fadd_noret_f64:901; GFX942: ; %bb.0:902; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)903; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.0904; GFX942-NEXT: ds_add_f64 v0, v[2:3]905; GFX942-NEXT: s_waitcnt lgkmcnt(0)906; GFX942-NEXT: s_setpc_b64 s[30:31]907;908; GFX11-LABEL: local_atomic_fadd_noret_f64:909; GFX11: ; %bb.0:910; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)911; GFX11-NEXT: ds_load_b64 v[1:2], v0912; GFX11-NEXT: s_mov_b32 s0, 0913; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start914; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1915; GFX11-NEXT: s_waitcnt lgkmcnt(0)916; GFX11-NEXT: v_add_f64 v[3:4], v[1:2], 4.0917; GFX11-NEXT: s_waitcnt_vscnt null, 0x0918; GFX11-NEXT: ds_cmpstore_rtn_b64 v[3:4], v0, v[3:4], v[1:2]919; GFX11-NEXT: s_waitcnt lgkmcnt(0)920; GFX11-NEXT: buffer_gl0_inv921; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]922; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v4923; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0924; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)925; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0926; GFX11-NEXT: s_cbranch_execnz .LBB6_1927; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end928; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0929; GFX11-NEXT: s_setpc_b64 s[30:31]930;931; GFX10-LABEL: local_atomic_fadd_noret_f64:932; GFX10: ; %bb.0:933; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)934; GFX10-NEXT: ds_read_b64 v[1:2], v0935; GFX10-NEXT: s_mov_b32 s4, 0936; GFX10-NEXT: .LBB6_1: ; %atomicrmw.start937; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1938; GFX10-NEXT: s_waitcnt lgkmcnt(0)939; GFX10-NEXT: v_add_f64 v[3:4], v[1:2], 4.0940; GFX10-NEXT: s_waitcnt_vscnt null, 0x0941; GFX10-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]942; GFX10-NEXT: s_waitcnt lgkmcnt(0)943; GFX10-NEXT: buffer_gl0_inv944; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]945; GFX10-NEXT: v_mov_b32_e32 v1, v3946; GFX10-NEXT: v_mov_b32_e32 v2, v4947; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4948; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4949; GFX10-NEXT: s_cbranch_execnz .LBB6_1950; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end951; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4952; GFX10-NEXT: s_setpc_b64 s[30:31]953;954; GFX90A-LABEL: local_atomic_fadd_noret_f64:955; GFX90A: ; %bb.0:956; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)957; GFX90A-NEXT: v_mov_b32_e32 v2, 0958; GFX90A-NEXT: v_mov_b32_e32 v3, 0x40100000959; GFX90A-NEXT: ds_add_f64 v0, v[2:3]960; GFX90A-NEXT: s_waitcnt lgkmcnt(0)961; GFX90A-NEXT: s_setpc_b64 s[30:31]962;963; GFX908-LABEL: local_atomic_fadd_noret_f64:964; GFX908: ; %bb.0:965; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)966; GFX908-NEXT: ds_read_b64 v[1:2], v0967; GFX908-NEXT: s_mov_b64 s[4:5], 0968; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start969; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1970; GFX908-NEXT: s_waitcnt lgkmcnt(0)971; GFX908-NEXT: v_add_f64 v[3:4], v[1:2], 4.0972; GFX908-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]973; GFX908-NEXT: s_waitcnt lgkmcnt(0)974; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]975; GFX908-NEXT: v_mov_b32_e32 v1, v3976; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]977; GFX908-NEXT: v_mov_b32_e32 v2, v4978; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]979; GFX908-NEXT: s_cbranch_execnz .LBB6_1980; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end981; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]982; GFX908-NEXT: s_setpc_b64 s[30:31]983;984; GFX8-LABEL: local_atomic_fadd_noret_f64:985; GFX8: ; %bb.0:986; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)987; GFX8-NEXT: s_mov_b32 m0, -1988; GFX8-NEXT: ds_read_b64 v[1:2], v0989; GFX8-NEXT: s_mov_b64 s[4:5], 0990; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start991; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1992; GFX8-NEXT: s_waitcnt lgkmcnt(0)993; GFX8-NEXT: v_add_f64 v[3:4], v[1:2], 4.0994; GFX8-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]995; GFX8-NEXT: s_waitcnt lgkmcnt(0)996; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]997; GFX8-NEXT: v_mov_b32_e32 v1, v3998; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]999; GFX8-NEXT: v_mov_b32_e32 v2, v41000; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1001; GFX8-NEXT: s_cbranch_execnz .LBB6_11002; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1003; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1004; GFX8-NEXT: s_setpc_b64 s[30:31]1005;1006; GFX7-LABEL: local_atomic_fadd_noret_f64:1007; GFX7: ; %bb.0:1008; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1009; GFX7-NEXT: s_mov_b32 m0, -11010; GFX7-NEXT: ds_read_b64 v[1:2], v01011; GFX7-NEXT: s_mov_b64 s[4:5], 01012; GFX7-NEXT: .LBB6_1: ; %atomicrmw.start1013; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11014; GFX7-NEXT: s_waitcnt lgkmcnt(0)1015; GFX7-NEXT: v_add_f64 v[3:4], v[1:2], 4.01016; GFX7-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]1017; GFX7-NEXT: s_waitcnt lgkmcnt(0)1018; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]1019; GFX7-NEXT: v_mov_b32_e32 v1, v31020; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1021; GFX7-NEXT: v_mov_b32_e32 v2, v41022; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1023; GFX7-NEXT: s_cbranch_execnz .LBB6_11024; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1025; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1026; GFX7-NEXT: s_setpc_b64 s[30:31]1027;1028; GFX6-LABEL: local_atomic_fadd_noret_f64:1029; GFX6: ; %bb.0:1030; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1031; GFX6-NEXT: s_mov_b32 m0, -11032; GFX6-NEXT: ds_read_b64 v[1:2], v01033; GFX6-NEXT: s_mov_b64 s[4:5], 01034; GFX6-NEXT: .LBB6_1: ; %atomicrmw.start1035; GFX6-NEXT: ; =>This Inner Loop Header: Depth=11036; GFX6-NEXT: s_waitcnt lgkmcnt(0)1037; GFX6-NEXT: v_add_f64 v[3:4], v[1:2], 4.01038; GFX6-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4]1039; GFX6-NEXT: s_waitcnt lgkmcnt(0)1040; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]1041; GFX6-NEXT: v_mov_b32_e32 v1, v31042; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1043; GFX6-NEXT: v_mov_b32_e32 v2, v41044; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]1045; GFX6-NEXT: s_cbranch_execnz .LBB6_11046; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end1047; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]1048; GFX6-NEXT: s_setpc_b64 s[30:31]1049 %result = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst1050 ret void1051}1052 1053define void @local_atomic_fadd_noret_f64__offset(ptr addrspace(3) %ptr) nounwind {1054; GFX12-LABEL: local_atomic_fadd_noret_f64__offset:1055; GFX12: ; %bb.0:1056; GFX12-NEXT: s_wait_loadcnt_dscnt 0x01057; GFX12-NEXT: s_wait_expcnt 0x01058; GFX12-NEXT: s_wait_samplecnt 0x01059; GFX12-NEXT: s_wait_bvhcnt 0x01060; GFX12-NEXT: s_wait_kmcnt 0x01061; GFX12-NEXT: ds_load_b64 v[1:2], v0 offset:655281062; GFX12-NEXT: s_mov_b32 s0, 01063; GFX12-NEXT: .LBB7_1: ; %atomicrmw.start1064; GFX12-NEXT: ; =>This Inner Loop Header: Depth=11065; GFX12-NEXT: s_wait_dscnt 0x01066; GFX12-NEXT: v_add_f64_e32 v[3:4], 4.0, v[1:2]1067; GFX12-NEXT: s_wait_storecnt 0x01068; GFX12-NEXT: ds_cmpstore_rtn_b64 v[3:4], v0, v[3:4], v[1:2] offset:655281069; GFX12-NEXT: s_wait_dscnt 0x01070; GFX12-NEXT: global_inv scope:SCOPE_SE1071; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]1072; GFX12-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v41073; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1074; GFX12-NEXT: s_or_b32 s0, vcc_lo, s01075; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)1076; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01077; GFX12-NEXT: s_cbranch_execnz .LBB7_11078; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end1079; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s01080; GFX12-NEXT: s_wait_loadcnt 0x01081; GFX12-NEXT: s_setpc_b64 s[30:31]1082;1083; GFX942-LABEL: local_atomic_fadd_noret_f64__offset:1084; GFX942: ; %bb.0:1085; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1086; GFX942-NEXT: v_mov_b64_e32 v[2:3], 4.01087; GFX942-NEXT: ds_add_f64 v0, v[2:3] offset:655281088; GFX942-NEXT: s_waitcnt lgkmcnt(0)1089; GFX942-NEXT: s_setpc_b64 s[30:31]1090;1091; GFX11-LABEL: local_atomic_fadd_noret_f64__offset:1092; GFX11: ; %bb.0:1093; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1094; GFX11-NEXT: ds_load_b64 v[1:2], v0 offset:655281095; GFX11-NEXT: s_mov_b32 s0, 01096; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start1097; GFX11-NEXT: ; =>This Inner Loop Header: Depth=11098; GFX11-NEXT: s_waitcnt lgkmcnt(0)1099; GFX11-NEXT: v_add_f64 v[3:4], v[1:2], 4.01100; GFX11-NEXT: s_waitcnt_vscnt null, 0x01101; GFX11-NEXT: ds_cmpstore_rtn_b64 v[3:4], v0, v[3:4], v[1:2] offset:655281102; GFX11-NEXT: s_waitcnt lgkmcnt(0)1103; GFX11-NEXT: buffer_gl0_inv1104; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]1105; GFX11-NEXT: v_dual_mov_b32 v1, v3 :: v_dual_mov_b32 v2, v41106; GFX11-NEXT: s_or_b32 s0, vcc_lo, s01107; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1108; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01109; GFX11-NEXT: s_cbranch_execnz .LBB7_11110; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end1111; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s01112; GFX11-NEXT: s_setpc_b64 s[30:31]1113;1114; GFX10-LABEL: local_atomic_fadd_noret_f64__offset:1115; GFX10: ; %bb.0:1116; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1117; GFX10-NEXT: ds_read_b64 v[1:2], v0 offset:655281118; GFX10-NEXT: s_mov_b32 s4, 01119; GFX10-NEXT: .LBB7_1: ; %atomicrmw.start1120; GFX10-NEXT: ; =>This Inner Loop Header: Depth=11121; GFX10-NEXT: s_waitcnt lgkmcnt(0)1122; GFX10-NEXT: v_add_f64 v[3:4], v[1:2], 4.01123; GFX10-NEXT: s_waitcnt_vscnt null, 0x01124; GFX10-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4] offset:655281125; GFX10-NEXT: s_waitcnt lgkmcnt(0)1126; GFX10-NEXT: buffer_gl0_inv1127; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[3:4], v[1:2]1128; GFX10-NEXT: v_mov_b32_e32 v1, v31129; GFX10-NEXT: v_mov_b32_e32 v2, v41130; GFX10-NEXT: s_or_b32 s4, vcc_lo, s41131; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s41132; GFX10-NEXT: s_cbranch_execnz .LBB7_11133; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end1134; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s41135; GFX10-NEXT: s_setpc_b64 s[30:31]1136;1137; GFX90A-LABEL: local_atomic_fadd_noret_f64__offset:1138; GFX90A: ; %bb.0:1139; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1140; GFX90A-NEXT: v_mov_b32_e32 v2, 01141; GFX90A-NEXT: v_mov_b32_e32 v3, 0x401000001142; GFX90A-NEXT: ds_add_f64 v0, v[2:3] offset:655281143; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1144; GFX90A-NEXT: s_setpc_b64 s[30:31]1145;1146; GFX908-LABEL: local_atomic_fadd_noret_f64__offset:1147; GFX908: ; %bb.0:1148; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1149; GFX908-NEXT: ds_read_b64 v[1:2], v0 offset:655281150; GFX908-NEXT: s_mov_b64 s[4:5], 01151; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start1152; GFX908-NEXT: ; =>This Inner Loop Header: Depth=11153; GFX908-NEXT: s_waitcnt lgkmcnt(0)1154; GFX908-NEXT: v_add_f64 v[3:4], v[1:2], 4.01155; GFX908-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4] offset:655281156; GFX908-NEXT: s_waitcnt lgkmcnt(0)1157; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]1158; GFX908-NEXT: v_mov_b32_e32 v1, v31159; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1160; GFX908-NEXT: v_mov_b32_e32 v2, v41161; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]1162; GFX908-NEXT: s_cbranch_execnz .LBB7_11163; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end1164; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]1165; GFX908-NEXT: s_setpc_b64 s[30:31]1166;1167; GFX8-LABEL: local_atomic_fadd_noret_f64__offset:1168; GFX8: ; %bb.0:1169; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1170; GFX8-NEXT: s_mov_b32 m0, -11171; GFX8-NEXT: ds_read_b64 v[1:2], v0 offset:655281172; GFX8-NEXT: s_mov_b64 s[4:5], 01173; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start1174; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11175; GFX8-NEXT: s_waitcnt lgkmcnt(0)1176; GFX8-NEXT: v_add_f64 v[3:4], v[1:2], 4.01177; GFX8-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4] offset:655281178; GFX8-NEXT: s_waitcnt lgkmcnt(0)1179; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]1180; GFX8-NEXT: v_mov_b32_e32 v1, v31181; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1182; GFX8-NEXT: v_mov_b32_e32 v2, v41183; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1184; GFX8-NEXT: s_cbranch_execnz .LBB7_11185; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1186; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1187; GFX8-NEXT: s_setpc_b64 s[30:31]1188;1189; GFX7-LABEL: local_atomic_fadd_noret_f64__offset:1190; GFX7: ; %bb.0:1191; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1192; GFX7-NEXT: s_mov_b32 m0, -11193; GFX7-NEXT: ds_read_b64 v[1:2], v0 offset:655281194; GFX7-NEXT: s_mov_b64 s[4:5], 01195; GFX7-NEXT: .LBB7_1: ; %atomicrmw.start1196; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11197; GFX7-NEXT: s_waitcnt lgkmcnt(0)1198; GFX7-NEXT: v_add_f64 v[3:4], v[1:2], 4.01199; GFX7-NEXT: ds_cmpst_rtn_b64 v[3:4], v0, v[1:2], v[3:4] offset:655281200; GFX7-NEXT: s_waitcnt lgkmcnt(0)1201; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]1202; GFX7-NEXT: v_mov_b32_e32 v1, v31203; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1204; GFX7-NEXT: v_mov_b32_e32 v2, v41205; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1206; GFX7-NEXT: s_cbranch_execnz .LBB7_11207; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1208; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1209; GFX7-NEXT: s_setpc_b64 s[30:31]1210;1211; GFX6-LABEL: local_atomic_fadd_noret_f64__offset:1212; GFX6: ; %bb.0:1213; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1214; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfff8, v01215; GFX6-NEXT: s_mov_b32 m0, -11216; GFX6-NEXT: ds_read_b64 v[0:1], v21217; GFX6-NEXT: s_mov_b64 s[4:5], 01218; GFX6-NEXT: .LBB7_1: ; %atomicrmw.start1219; GFX6-NEXT: ; =>This Inner Loop Header: Depth=11220; GFX6-NEXT: s_waitcnt lgkmcnt(0)1221; GFX6-NEXT: v_add_f64 v[3:4], v[0:1], 4.01222; GFX6-NEXT: ds_cmpst_rtn_b64 v[3:4], v2, v[0:1], v[3:4]1223; GFX6-NEXT: s_waitcnt lgkmcnt(0)1224; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[0:1]1225; GFX6-NEXT: v_mov_b32_e32 v0, v31226; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1227; GFX6-NEXT: v_mov_b32_e32 v1, v41228; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]1229; GFX6-NEXT: s_cbranch_execnz .LBB7_11230; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end1231; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]1232; GFX6-NEXT: s_setpc_b64 s[30:31]1233 %gep = getelementptr double, ptr addrspace(3) %ptr, i32 81911234 %unused = atomicrmw fadd ptr addrspace(3) %gep, double 4.0 seq_cst1235 ret void1236}1237 1238; --------------------------------------------------------------------1239; half1240; --------------------------------------------------------------------1241 1242define half @local_atomic_fadd_ret_f16(ptr addrspace(3) %ptr) nounwind {1243; GFX12-TRUE16-LABEL: local_atomic_fadd_ret_f16:1244; GFX12-TRUE16: ; %bb.0:1245; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x01246; GFX12-TRUE16-NEXT: s_wait_expcnt 0x01247; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x01248; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x01249; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x01250; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, -4, v01251; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v01252; GFX12-TRUE16-NEXT: s_mov_b32 s0, 01253; GFX12-TRUE16-NEXT: ds_load_b32 v3, v11254; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff1255; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 24, v01256; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)1257; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v21258; GFX12-TRUE16-NEXT: .LBB8_1: ; %atomicrmw.start1259; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=11260; GFX12-TRUE16-NEXT: s_wait_dscnt 0x01261; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v31262; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)1263; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v41264; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 01265; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, 4.0, v3.l1266; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1267; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v31268; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v31269; GFX12-TRUE16-NEXT: s_wait_storecnt 0x01270; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v41271; GFX12-TRUE16-NEXT: s_wait_dscnt 0x01272; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE1273; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v41274; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)1275; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s01276; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)1277; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01278; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB8_11279; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end1280; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s01281; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v31282; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x01283; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]1284;1285; GFX12-FAKE16-LABEL: local_atomic_fadd_ret_f16:1286; GFX12-FAKE16: ; %bb.0:1287; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x01288; GFX12-FAKE16-NEXT: s_wait_expcnt 0x01289; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x01290; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x01291; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x01292; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, -4, v01293; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 3, v01294; GFX12-FAKE16-NEXT: s_mov_b32 s0, 01295; GFX12-FAKE16-NEXT: ds_load_b32 v2, v11296; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff1297; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 24, v01298; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)1299; GFX12-FAKE16-NEXT: v_not_b32_e32 v3, v31300; GFX12-FAKE16-NEXT: .LBB8_1: ; %atomicrmw.start1301; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=11302; GFX12-FAKE16-NEXT: s_wait_dscnt 0x01303; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v21304; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1305; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v0, v41306; GFX12-FAKE16-NEXT: v_add_f16_e32 v2, 4.0, v21307; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1308; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v21309; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v0, v21310; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1311; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v4, v3, v21312; GFX12-FAKE16-NEXT: s_wait_storecnt 0x01313; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v1, v2, v41314; GFX12-FAKE16-NEXT: s_wait_dscnt 0x01315; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE1316; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v41317; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)1318; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s01319; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)1320; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01321; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB8_11322; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end1323; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s01324; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v0, v21325; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x01326; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]1327;1328; GFX942-LABEL: local_atomic_fadd_ret_f16:1329; GFX942: ; %bb.0:1330; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1331; GFX942-NEXT: v_and_b32_e32 v1, -4, v01332; GFX942-NEXT: ds_read_b32 v3, v11333; GFX942-NEXT: v_lshlrev_b32_e32 v2, 3, v01334; GFX942-NEXT: s_mov_b32 s0, 0xffff1335; GFX942-NEXT: v_and_b32_e32 v0, 24, v21336; GFX942-NEXT: v_lshlrev_b32_e64 v2, v2, s01337; GFX942-NEXT: v_not_b32_e32 v2, v21338; GFX942-NEXT: s_mov_b64 s[0:1], 01339; GFX942-NEXT: .LBB8_1: ; %atomicrmw.start1340; GFX942-NEXT: ; =>This Inner Loop Header: Depth=11341; GFX942-NEXT: s_waitcnt lgkmcnt(0)1342; GFX942-NEXT: v_lshrrev_b32_e32 v4, v0, v31343; GFX942-NEXT: v_add_f16_e32 v4, 4.0, v41344; GFX942-NEXT: v_lshlrev_b32_e32 v4, v0, v41345; GFX942-NEXT: v_and_or_b32 v4, v3, v2, v41346; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v41347; GFX942-NEXT: s_waitcnt lgkmcnt(0)1348; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v31349; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]1350; GFX942-NEXT: v_mov_b32_e32 v3, v41351; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]1352; GFX942-NEXT: s_cbranch_execnz .LBB8_11353; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end1354; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]1355; GFX942-NEXT: v_lshrrev_b32_e32 v0, v0, v41356; GFX942-NEXT: s_setpc_b64 s[30:31]1357;1358; GFX11-TRUE16-LABEL: local_atomic_fadd_ret_f16:1359; GFX11-TRUE16: ; %bb.0:1360; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1361; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, -4, v01362; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v01363; GFX11-TRUE16-NEXT: s_mov_b32 s0, 01364; GFX11-TRUE16-NEXT: ds_load_b32 v3, v11365; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff1366; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 24, v01367; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)1368; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v21369; GFX11-TRUE16-NEXT: .LBB8_1: ; %atomicrmw.start1370; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=11371; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)1372; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v31373; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)1374; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v41375; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 01376; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, 4.0, v3.l1377; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1378; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v31379; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v31380; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x01381; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v41382; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)1383; GFX11-TRUE16-NEXT: buffer_gl0_inv1384; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v41385; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s01386; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1387; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01388; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB8_11389; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end1390; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s01391; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v31392; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]1393;1394; GFX11-FAKE16-LABEL: local_atomic_fadd_ret_f16:1395; GFX11-FAKE16: ; %bb.0:1396; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1397; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, -4, v01398; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 3, v01399; GFX11-FAKE16-NEXT: s_mov_b32 s0, 01400; GFX11-FAKE16-NEXT: ds_load_b32 v2, v11401; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff1402; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 24, v01403; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)1404; GFX11-FAKE16-NEXT: v_not_b32_e32 v3, v31405; GFX11-FAKE16-NEXT: .LBB8_1: ; %atomicrmw.start1406; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=11407; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)1408; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v21409; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1410; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v0, v41411; GFX11-FAKE16-NEXT: v_add_f16_e32 v2, 4.0, v21412; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1413; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v21414; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v0, v21415; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1416; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v4, v3, v21417; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x01418; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v1, v2, v41419; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)1420; GFX11-FAKE16-NEXT: buffer_gl0_inv1421; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v41422; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s01423; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1424; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01425; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB8_11426; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end1427; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s01428; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v0, v21429; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]1430;1431; GFX10-LABEL: local_atomic_fadd_ret_f16:1432; GFX10: ; %bb.0:1433; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1434; GFX10-NEXT: v_and_b32_e32 v1, -4, v01435; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v01436; GFX10-NEXT: s_mov_b32 s4, 01437; GFX10-NEXT: ds_read_b32 v2, v11438; GFX10-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff1439; GFX10-NEXT: v_and_b32_e32 v0, 24, v01440; GFX10-NEXT: v_not_b32_e32 v3, v31441; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start1442; GFX10-NEXT: ; =>This Inner Loop Header: Depth=11443; GFX10-NEXT: s_waitcnt lgkmcnt(0)1444; GFX10-NEXT: v_mov_b32_e32 v4, v21445; GFX10-NEXT: v_lshrrev_b32_e32 v2, v0, v41446; GFX10-NEXT: v_add_f16_e32 v2, 4.0, v21447; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_01448; GFX10-NEXT: v_and_or_b32 v2, v4, v3, v21449; GFX10-NEXT: s_waitcnt_vscnt null, 0x01450; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v21451; GFX10-NEXT: s_waitcnt lgkmcnt(0)1452; GFX10-NEXT: buffer_gl0_inv1453; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v41454; GFX10-NEXT: s_or_b32 s4, vcc_lo, s41455; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s41456; GFX10-NEXT: s_cbranch_execnz .LBB8_11457; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end1458; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s41459; GFX10-NEXT: v_lshrrev_b32_e32 v0, v0, v21460; GFX10-NEXT: s_setpc_b64 s[30:31]1461;1462; GFX90A-LABEL: local_atomic_fadd_ret_f16:1463; GFX90A: ; %bb.0:1464; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1465; GFX90A-NEXT: v_and_b32_e32 v1, -4, v01466; GFX90A-NEXT: ds_read_b32 v3, v11467; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 3, v01468; GFX90A-NEXT: s_mov_b32 s4, 0xffff1469; GFX90A-NEXT: v_and_b32_e32 v0, 24, v21470; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v2, s41471; GFX90A-NEXT: v_not_b32_e32 v2, v21472; GFX90A-NEXT: s_mov_b64 s[4:5], 01473; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start1474; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11475; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1476; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v0, v31477; GFX90A-NEXT: v_add_f16_e32 v4, 4.0, v41478; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v0, v41479; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v41480; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v41481; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1482; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v31483; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1484; GFX90A-NEXT: v_mov_b32_e32 v3, v41485; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1486; GFX90A-NEXT: s_cbranch_execnz .LBB8_11487; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1488; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1489; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v0, v41490; GFX90A-NEXT: s_setpc_b64 s[30:31]1491;1492; GFX908-LABEL: local_atomic_fadd_ret_f16:1493; GFX908: ; %bb.0:1494; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1495; GFX908-NEXT: v_and_b32_e32 v1, -4, v01496; GFX908-NEXT: ds_read_b32 v2, v11497; GFX908-NEXT: v_lshlrev_b32_e32 v3, 3, v01498; GFX908-NEXT: s_mov_b32 s4, 0xffff1499; GFX908-NEXT: v_and_b32_e32 v0, 24, v31500; GFX908-NEXT: v_lshlrev_b32_e64 v3, v3, s41501; GFX908-NEXT: v_not_b32_e32 v3, v31502; GFX908-NEXT: s_mov_b64 s[4:5], 01503; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start1504; GFX908-NEXT: ; =>This Inner Loop Header: Depth=11505; GFX908-NEXT: s_waitcnt lgkmcnt(0)1506; GFX908-NEXT: v_mov_b32_e32 v4, v21507; GFX908-NEXT: v_lshrrev_b32_e32 v2, v0, v41508; GFX908-NEXT: v_add_f16_e32 v2, 4.0, v21509; GFX908-NEXT: v_lshlrev_b32_e32 v2, v0, v21510; GFX908-NEXT: v_and_or_b32 v2, v4, v3, v21511; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v21512; GFX908-NEXT: s_waitcnt lgkmcnt(0)1513; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v41514; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1515; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]1516; GFX908-NEXT: s_cbranch_execnz .LBB8_11517; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end1518; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]1519; GFX908-NEXT: v_lshrrev_b32_e32 v0, v0, v21520; GFX908-NEXT: s_setpc_b64 s[30:31]1521;1522; GFX8-LABEL: local_atomic_fadd_ret_f16:1523; GFX8: ; %bb.0:1524; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1525; GFX8-NEXT: v_and_b32_e32 v1, -4, v01526; GFX8-NEXT: s_mov_b32 m0, -11527; GFX8-NEXT: ds_read_b32 v3, v11528; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v01529; GFX8-NEXT: s_mov_b32 s4, 0xffff1530; GFX8-NEXT: v_and_b32_e32 v0, 24, v21531; GFX8-NEXT: v_lshlrev_b32_e64 v2, v2, s41532; GFX8-NEXT: v_not_b32_e32 v2, v21533; GFX8-NEXT: s_mov_b64 s[4:5], 01534; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start1535; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11536; GFX8-NEXT: s_waitcnt lgkmcnt(0)1537; GFX8-NEXT: v_mov_b32_e32 v4, v31538; GFX8-NEXT: v_lshrrev_b32_e32 v3, v0, v41539; GFX8-NEXT: v_add_f16_e32 v3, 4.0, v31540; GFX8-NEXT: v_and_b32_e32 v5, v4, v21541; GFX8-NEXT: v_lshlrev_b32_e32 v3, v0, v31542; GFX8-NEXT: v_or_b32_e32 v3, v5, v31543; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v31544; GFX8-NEXT: s_waitcnt lgkmcnt(0)1545; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v41546; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1547; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1548; GFX8-NEXT: s_cbranch_execnz .LBB8_11549; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1550; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1551; GFX8-NEXT: v_lshrrev_b32_e32 v0, v0, v31552; GFX8-NEXT: s_setpc_b64 s[30:31]1553;1554; GFX7-LABEL: local_atomic_fadd_ret_f16:1555; GFX7: ; %bb.0:1556; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1557; GFX7-NEXT: v_and_b32_e32 v1, -4, v01558; GFX7-NEXT: s_mov_b32 m0, -11559; GFX7-NEXT: ds_read_b32 v3, v11560; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v01561; GFX7-NEXT: v_and_b32_e32 v0, 24, v21562; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v21563; GFX7-NEXT: v_not_b32_e32 v2, v21564; GFX7-NEXT: s_mov_b64 s[4:5], 01565; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start1566; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11567; GFX7-NEXT: s_waitcnt lgkmcnt(0)1568; GFX7-NEXT: v_mov_b32_e32 v4, v31569; GFX7-NEXT: v_lshrrev_b32_e32 v3, v0, v41570; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v31571; GFX7-NEXT: v_and_b32_e32 v5, v4, v21572; GFX7-NEXT: v_add_f32_e32 v3, 4.0, v31573; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v31574; GFX7-NEXT: v_lshlrev_b32_e32 v3, v0, v31575; GFX7-NEXT: v_or_b32_e32 v3, v5, v31576; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v31577; GFX7-NEXT: s_waitcnt lgkmcnt(0)1578; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v41579; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1580; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1581; GFX7-NEXT: s_cbranch_execnz .LBB8_11582; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1583; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1584; GFX7-NEXT: v_lshrrev_b32_e32 v0, v0, v31585; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v01586; GFX7-NEXT: s_setpc_b64 s[30:31]1587;1588; GFX6-LABEL: local_atomic_fadd_ret_f16:1589; GFX6: ; %bb.0:1590; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1591; GFX6-NEXT: v_and_b32_e32 v1, -4, v01592; GFX6-NEXT: s_mov_b32 m0, -11593; GFX6-NEXT: ds_read_b32 v3, v11594; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v01595; GFX6-NEXT: v_and_b32_e32 v0, 24, v21596; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v21597; GFX6-NEXT: v_not_b32_e32 v2, v21598; GFX6-NEXT: s_mov_b64 s[4:5], 01599; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start1600; GFX6-NEXT: ; =>This Inner Loop Header: Depth=11601; GFX6-NEXT: s_waitcnt lgkmcnt(0)1602; GFX6-NEXT: v_mov_b32_e32 v4, v31603; GFX6-NEXT: v_lshrrev_b32_e32 v3, v0, v41604; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v31605; GFX6-NEXT: v_and_b32_e32 v5, v4, v21606; GFX6-NEXT: v_add_f32_e32 v3, 4.0, v31607; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v31608; GFX6-NEXT: v_lshlrev_b32_e32 v3, v0, v31609; GFX6-NEXT: v_or_b32_e32 v3, v5, v31610; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v31611; GFX6-NEXT: s_waitcnt lgkmcnt(0)1612; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v41613; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1614; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]1615; GFX6-NEXT: s_cbranch_execnz .LBB8_11616; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end1617; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]1618; GFX6-NEXT: v_lshrrev_b32_e32 v0, v0, v31619; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v01620; GFX6-NEXT: s_setpc_b64 s[30:31]1621 %result = atomicrmw fadd ptr addrspace(3) %ptr, half 4.0 seq_cst1622 ret half %result1623}1624 1625define half @local_atomic_fadd_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {1626; GFX12-TRUE16-LABEL: local_atomic_fadd_ret_f16__offset:1627; GFX12-TRUE16: ; %bb.0:1628; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x01629; GFX12-TRUE16-NEXT: s_wait_expcnt 0x01630; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x01631; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x01632; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x01633; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v01634; GFX12-TRUE16-NEXT: s_mov_b32 s0, 01635; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)1636; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v11637; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 3, v11638; GFX12-TRUE16-NEXT: ds_load_b32 v3, v01639; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v11640; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff1641; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1642; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v21643; GFX12-TRUE16-NEXT: .LBB9_1: ; %atomicrmw.start1644; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=11645; GFX12-TRUE16-NEXT: s_wait_dscnt 0x01646; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v31647; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)1648; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v41649; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 01650; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, 4.0, v3.l1651; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1652; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v31653; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v31654; GFX12-TRUE16-NEXT: s_wait_storecnt 0x01655; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v41656; GFX12-TRUE16-NEXT: s_wait_dscnt 0x01657; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE1658; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v41659; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)1660; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s01661; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)1662; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01663; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB9_11664; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end1665; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s01666; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v31667; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x01668; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]1669;1670; GFX12-FAKE16-LABEL: local_atomic_fadd_ret_f16__offset:1671; GFX12-FAKE16: ; %bb.0:1672; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x01673; GFX12-FAKE16-NEXT: s_wait_expcnt 0x01674; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x01675; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x01676; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x01677; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v01678; GFX12-FAKE16-NEXT: s_mov_b32 s0, 01679; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)1680; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v11681; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 3, v11682; GFX12-FAKE16-NEXT: ds_load_b32 v3, v01683; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 3, v11684; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff1685; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1686; GFX12-FAKE16-NEXT: v_not_b32_e32 v2, v21687; GFX12-FAKE16-NEXT: .LBB9_1: ; %atomicrmw.start1688; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=11689; GFX12-FAKE16-NEXT: s_wait_dscnt 0x01690; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v31691; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1692; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v1, v41693; GFX12-FAKE16-NEXT: v_add_f16_e32 v3, 4.0, v31694; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1695; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v31696; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v1, v31697; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1698; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v2, v31699; GFX12-FAKE16-NEXT: s_wait_storecnt 0x01700; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v41701; GFX12-FAKE16-NEXT: s_wait_dscnt 0x01702; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE1703; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v41704; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)1705; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s01706; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)1707; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01708; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB9_11709; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end1710; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s01711; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v1, v31712; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x01713; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]1714;1715; GFX942-LABEL: local_atomic_fadd_ret_f16__offset:1716; GFX942: ; %bb.0:1717; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1718; GFX942-NEXT: v_add_u32_e32 v0, 0xfffe, v01719; GFX942-NEXT: v_and_b32_e32 v1, -4, v01720; GFX942-NEXT: ds_read_b32 v3, v11721; GFX942-NEXT: v_and_b32_e32 v0, 3, v01722; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v01723; GFX942-NEXT: s_mov_b32 s0, 0xffff1724; GFX942-NEXT: v_lshlrev_b32_e64 v2, v0, s01725; GFX942-NEXT: v_not_b32_e32 v2, v21726; GFX942-NEXT: s_mov_b64 s[0:1], 01727; GFX942-NEXT: .LBB9_1: ; %atomicrmw.start1728; GFX942-NEXT: ; =>This Inner Loop Header: Depth=11729; GFX942-NEXT: s_waitcnt lgkmcnt(0)1730; GFX942-NEXT: v_lshrrev_b32_e32 v4, v0, v31731; GFX942-NEXT: v_add_f16_e32 v4, 4.0, v41732; GFX942-NEXT: v_lshlrev_b32_e32 v4, v0, v41733; GFX942-NEXT: v_and_or_b32 v4, v3, v2, v41734; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v41735; GFX942-NEXT: s_waitcnt lgkmcnt(0)1736; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v31737; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]1738; GFX942-NEXT: v_mov_b32_e32 v3, v41739; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]1740; GFX942-NEXT: s_cbranch_execnz .LBB9_11741; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end1742; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]1743; GFX942-NEXT: v_lshrrev_b32_e32 v0, v0, v41744; GFX942-NEXT: s_setpc_b64 s[30:31]1745;1746; GFX11-TRUE16-LABEL: local_atomic_fadd_ret_f16__offset:1747; GFX11-TRUE16: ; %bb.0:1748; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1749; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v01750; GFX11-TRUE16-NEXT: s_mov_b32 s0, 01751; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)1752; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v11753; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 3, v11754; GFX11-TRUE16-NEXT: ds_load_b32 v3, v01755; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v11756; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff1757; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1758; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v21759; GFX11-TRUE16-NEXT: .LBB9_1: ; %atomicrmw.start1760; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=11761; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)1762; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v31763; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)1764; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v41765; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 01766; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, 4.0, v3.l1767; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1768; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v31769; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v31770; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x01771; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v41772; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)1773; GFX11-TRUE16-NEXT: buffer_gl0_inv1774; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v41775; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s01776; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1777; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01778; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB9_11779; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end1780; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s01781; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v31782; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]1783;1784; GFX11-FAKE16-LABEL: local_atomic_fadd_ret_f16__offset:1785; GFX11-FAKE16: ; %bb.0:1786; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1787; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v01788; GFX11-FAKE16-NEXT: s_mov_b32 s0, 01789; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)1790; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v11791; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 3, v11792; GFX11-FAKE16-NEXT: ds_load_b32 v3, v01793; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 3, v11794; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff1795; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1796; GFX11-FAKE16-NEXT: v_not_b32_e32 v2, v21797; GFX11-FAKE16-NEXT: .LBB9_1: ; %atomicrmw.start1798; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=11799; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)1800; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v31801; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1802; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v1, v41803; GFX11-FAKE16-NEXT: v_add_f16_e32 v3, 4.0, v31804; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)1805; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v31806; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v1, v31807; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)1808; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v2, v31809; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x01810; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v41811; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)1812; GFX11-FAKE16-NEXT: buffer_gl0_inv1813; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v41814; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s01815; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)1816; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s01817; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB9_11818; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end1819; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s01820; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v1, v31821; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]1822;1823; GFX10-LABEL: local_atomic_fadd_ret_f16__offset:1824; GFX10: ; %bb.0:1825; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1826; GFX10-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v01827; GFX10-NEXT: s_mov_b32 s4, 01828; GFX10-NEXT: v_and_b32_e32 v0, -4, v11829; GFX10-NEXT: v_and_b32_e32 v1, 3, v11830; GFX10-NEXT: ds_read_b32 v2, v01831; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v11832; GFX10-NEXT: v_lshlrev_b32_e64 v3, v1, 0xffff1833; GFX10-NEXT: v_not_b32_e32 v3, v31834; GFX10-NEXT: .LBB9_1: ; %atomicrmw.start1835; GFX10-NEXT: ; =>This Inner Loop Header: Depth=11836; GFX10-NEXT: s_waitcnt lgkmcnt(0)1837; GFX10-NEXT: v_mov_b32_e32 v4, v21838; GFX10-NEXT: v_lshrrev_b32_e32 v2, v1, v41839; GFX10-NEXT: v_add_f16_e32 v2, 4.0, v21840; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_01841; GFX10-NEXT: v_and_or_b32 v2, v4, v3, v21842; GFX10-NEXT: s_waitcnt_vscnt null, 0x01843; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v21844; GFX10-NEXT: s_waitcnt lgkmcnt(0)1845; GFX10-NEXT: buffer_gl0_inv1846; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v41847; GFX10-NEXT: s_or_b32 s4, vcc_lo, s41848; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s41849; GFX10-NEXT: s_cbranch_execnz .LBB9_11850; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end1851; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s41852; GFX10-NEXT: v_lshrrev_b32_e32 v0, v1, v21853; GFX10-NEXT: s_setpc_b64 s[30:31]1854;1855; GFX90A-LABEL: local_atomic_fadd_ret_f16__offset:1856; GFX90A: ; %bb.0:1857; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1858; GFX90A-NEXT: v_add_u32_e32 v0, 0xfffe, v01859; GFX90A-NEXT: v_and_b32_e32 v1, -4, v01860; GFX90A-NEXT: ds_read_b32 v3, v11861; GFX90A-NEXT: v_and_b32_e32 v0, 3, v01862; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 3, v01863; GFX90A-NEXT: s_mov_b32 s4, 0xffff1864; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v0, s41865; GFX90A-NEXT: v_not_b32_e32 v2, v21866; GFX90A-NEXT: s_mov_b64 s[4:5], 01867; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start1868; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=11869; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1870; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v0, v31871; GFX90A-NEXT: v_add_f16_e32 v4, 4.0, v41872; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v0, v41873; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v41874; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v41875; GFX90A-NEXT: s_waitcnt lgkmcnt(0)1876; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v31877; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1878; GFX90A-NEXT: v_mov_b32_e32 v3, v41879; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]1880; GFX90A-NEXT: s_cbranch_execnz .LBB9_11881; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end1882; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]1883; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v0, v41884; GFX90A-NEXT: s_setpc_b64 s[30:31]1885;1886; GFX908-LABEL: local_atomic_fadd_ret_f16__offset:1887; GFX908: ; %bb.0:1888; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1889; GFX908-NEXT: v_add_u32_e32 v0, 0xfffe, v01890; GFX908-NEXT: v_and_b32_e32 v1, -4, v01891; GFX908-NEXT: ds_read_b32 v2, v11892; GFX908-NEXT: v_and_b32_e32 v0, 3, v01893; GFX908-NEXT: v_lshlrev_b32_e32 v0, 3, v01894; GFX908-NEXT: s_mov_b32 s4, 0xffff1895; GFX908-NEXT: v_lshlrev_b32_e64 v3, v0, s41896; GFX908-NEXT: v_not_b32_e32 v3, v31897; GFX908-NEXT: s_mov_b64 s[4:5], 01898; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start1899; GFX908-NEXT: ; =>This Inner Loop Header: Depth=11900; GFX908-NEXT: s_waitcnt lgkmcnt(0)1901; GFX908-NEXT: v_mov_b32_e32 v4, v21902; GFX908-NEXT: v_lshrrev_b32_e32 v2, v0, v41903; GFX908-NEXT: v_add_f16_e32 v2, 4.0, v21904; GFX908-NEXT: v_lshlrev_b32_e32 v2, v0, v21905; GFX908-NEXT: v_and_or_b32 v2, v4, v3, v21906; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v1, v4, v21907; GFX908-NEXT: s_waitcnt lgkmcnt(0)1908; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v41909; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1910; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]1911; GFX908-NEXT: s_cbranch_execnz .LBB9_11912; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end1913; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]1914; GFX908-NEXT: v_lshrrev_b32_e32 v0, v0, v21915; GFX908-NEXT: s_setpc_b64 s[30:31]1916;1917; GFX8-LABEL: local_atomic_fadd_ret_f16__offset:1918; GFX8: ; %bb.0:1919; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1920; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffe, v01921; GFX8-NEXT: v_and_b32_e32 v1, -4, v01922; GFX8-NEXT: s_mov_b32 m0, -11923; GFX8-NEXT: ds_read_b32 v3, v11924; GFX8-NEXT: v_and_b32_e32 v0, 3, v01925; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v01926; GFX8-NEXT: s_mov_b32 s4, 0xffff1927; GFX8-NEXT: v_lshlrev_b32_e64 v2, v0, s41928; GFX8-NEXT: v_not_b32_e32 v2, v21929; GFX8-NEXT: s_mov_b64 s[4:5], 01930; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start1931; GFX8-NEXT: ; =>This Inner Loop Header: Depth=11932; GFX8-NEXT: s_waitcnt lgkmcnt(0)1933; GFX8-NEXT: v_mov_b32_e32 v4, v31934; GFX8-NEXT: v_lshrrev_b32_e32 v3, v0, v41935; GFX8-NEXT: v_add_f16_e32 v3, 4.0, v31936; GFX8-NEXT: v_and_b32_e32 v5, v4, v21937; GFX8-NEXT: v_lshlrev_b32_e32 v3, v0, v31938; GFX8-NEXT: v_or_b32_e32 v3, v5, v31939; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v31940; GFX8-NEXT: s_waitcnt lgkmcnt(0)1941; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v41942; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1943; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]1944; GFX8-NEXT: s_cbranch_execnz .LBB9_11945; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end1946; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]1947; GFX8-NEXT: v_lshrrev_b32_e32 v0, v0, v31948; GFX8-NEXT: s_setpc_b64 s[30:31]1949;1950; GFX7-LABEL: local_atomic_fadd_ret_f16__offset:1951; GFX7: ; %bb.0:1952; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1953; GFX7-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v01954; GFX7-NEXT: v_and_b32_e32 v0, -4, v11955; GFX7-NEXT: s_mov_b32 m0, -11956; GFX7-NEXT: ds_read_b32 v3, v01957; GFX7-NEXT: v_and_b32_e32 v1, 3, v11958; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v11959; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v11960; GFX7-NEXT: v_not_b32_e32 v2, v21961; GFX7-NEXT: s_mov_b64 s[4:5], 01962; GFX7-NEXT: .LBB9_1: ; %atomicrmw.start1963; GFX7-NEXT: ; =>This Inner Loop Header: Depth=11964; GFX7-NEXT: s_waitcnt lgkmcnt(0)1965; GFX7-NEXT: v_mov_b32_e32 v4, v31966; GFX7-NEXT: v_lshrrev_b32_e32 v3, v1, v41967; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v31968; GFX7-NEXT: v_and_b32_e32 v5, v4, v21969; GFX7-NEXT: v_add_f32_e32 v3, 4.0, v31970; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v31971; GFX7-NEXT: v_lshlrev_b32_e32 v3, v1, v31972; GFX7-NEXT: v_or_b32_e32 v3, v5, v31973; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v31974; GFX7-NEXT: s_waitcnt lgkmcnt(0)1975; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v41976; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]1977; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]1978; GFX7-NEXT: s_cbranch_execnz .LBB9_11979; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end1980; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]1981; GFX7-NEXT: v_lshrrev_b32_e32 v0, v1, v31982; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v01983; GFX7-NEXT: s_setpc_b64 s[30:31]1984;1985; GFX6-LABEL: local_atomic_fadd_ret_f16__offset:1986; GFX6: ; %bb.0:1987; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)1988; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v01989; GFX6-NEXT: v_and_b32_e32 v0, -4, v11990; GFX6-NEXT: s_mov_b32 m0, -11991; GFX6-NEXT: ds_read_b32 v3, v01992; GFX6-NEXT: v_and_b32_e32 v1, 3, v11993; GFX6-NEXT: v_lshlrev_b32_e32 v1, 3, v11994; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v11995; GFX6-NEXT: v_not_b32_e32 v2, v21996; GFX6-NEXT: s_mov_b64 s[4:5], 01997; GFX6-NEXT: .LBB9_1: ; %atomicrmw.start1998; GFX6-NEXT: ; =>This Inner Loop Header: Depth=11999; GFX6-NEXT: s_waitcnt lgkmcnt(0)2000; GFX6-NEXT: v_mov_b32_e32 v4, v32001; GFX6-NEXT: v_lshrrev_b32_e32 v3, v1, v42002; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v32003; GFX6-NEXT: v_and_b32_e32 v5, v4, v22004; GFX6-NEXT: v_add_f32_e32 v3, 4.0, v32005; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v32006; GFX6-NEXT: v_lshlrev_b32_e32 v3, v1, v32007; GFX6-NEXT: v_or_b32_e32 v3, v5, v32008; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v32009; GFX6-NEXT: s_waitcnt lgkmcnt(0)2010; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v42011; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2012; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]2013; GFX6-NEXT: s_cbranch_execnz .LBB9_12014; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end2015; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]2016; GFX6-NEXT: v_lshrrev_b32_e32 v0, v1, v32017; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v02018; GFX6-NEXT: s_setpc_b64 s[30:31]2019 %gep = getelementptr half, ptr addrspace(3) %ptr, i32 327672020 %result = atomicrmw fadd ptr addrspace(3) %gep, half 4.0 seq_cst2021 ret half %result2022}2023 2024define void @local_atomic_fadd_noret_f16(ptr addrspace(3) %ptr) nounwind {2025; GFX12-TRUE16-LABEL: local_atomic_fadd_noret_f16:2026; GFX12-TRUE16: ; %bb.0:2027; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x02028; GFX12-TRUE16-NEXT: s_wait_expcnt 0x02029; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x02030; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x02031; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x02032; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, -4, v02033; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v02034; GFX12-TRUE16-NEXT: s_mov_b32 s0, 02035; GFX12-TRUE16-NEXT: ds_load_b32 v2, v12036; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff2037; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 24, v02038; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)2039; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v32040; GFX12-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start2041; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=12042; GFX12-TRUE16-NEXT: s_wait_dscnt 0x02043; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)2044; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v22045; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 02046; GFX12-TRUE16-NEXT: v_add_f16_e32 v4.l, 4.0, v4.l2047; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2048; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v42049; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v42050; GFX12-TRUE16-NEXT: s_wait_storecnt 0x02051; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v22052; GFX12-TRUE16-NEXT: s_wait_dscnt 0x02053; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE2054; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v22055; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v42056; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)2057; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s02058; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)2059; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02060; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB10_12061; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end2062; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s02063; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x02064; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]2065;2066; GFX12-FAKE16-LABEL: local_atomic_fadd_noret_f16:2067; GFX12-FAKE16: ; %bb.0:2068; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x02069; GFX12-FAKE16-NEXT: s_wait_expcnt 0x02070; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x02071; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x02072; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x02073; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, -4, v02074; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 3, v02075; GFX12-FAKE16-NEXT: s_mov_b32 s0, 02076; GFX12-FAKE16-NEXT: ds_load_b32 v2, v12077; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff2078; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 24, v02079; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)2080; GFX12-FAKE16-NEXT: v_not_b32_e32 v3, v32081; GFX12-FAKE16-NEXT: .LBB10_1: ; %atomicrmw.start2082; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=12083; GFX12-FAKE16-NEXT: s_wait_dscnt 0x02084; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2085; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v0, v22086; GFX12-FAKE16-NEXT: v_add_f16_e32 v4, 4.0, v42087; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2088; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v42089; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v0, v42090; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2091; GFX12-FAKE16-NEXT: v_and_or_b32 v4, v2, v3, v42092; GFX12-FAKE16-NEXT: s_wait_storecnt 0x02093; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v22094; GFX12-FAKE16-NEXT: s_wait_dscnt 0x02095; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE2096; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v22097; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v42098; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)2099; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s02100; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)2101; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02102; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB10_12103; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end2104; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s02105; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x02106; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]2107;2108; GFX942-LABEL: local_atomic_fadd_noret_f16:2109; GFX942: ; %bb.0:2110; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2111; GFX942-NEXT: v_and_b32_e32 v1, -4, v02112; GFX942-NEXT: ds_read_b32 v3, v12113; GFX942-NEXT: v_lshlrev_b32_e32 v2, 3, v02114; GFX942-NEXT: s_mov_b32 s0, 0xffff2115; GFX942-NEXT: v_and_b32_e32 v0, 24, v22116; GFX942-NEXT: v_lshlrev_b32_e64 v2, v2, s02117; GFX942-NEXT: v_not_b32_e32 v2, v22118; GFX942-NEXT: s_mov_b64 s[0:1], 02119; GFX942-NEXT: .LBB10_1: ; %atomicrmw.start2120; GFX942-NEXT: ; =>This Inner Loop Header: Depth=12121; GFX942-NEXT: s_waitcnt lgkmcnt(0)2122; GFX942-NEXT: v_lshrrev_b32_e32 v4, v0, v32123; GFX942-NEXT: v_add_f16_e32 v4, 4.0, v42124; GFX942-NEXT: v_lshlrev_b32_e32 v4, v0, v42125; GFX942-NEXT: v_and_or_b32 v4, v3, v2, v42126; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v42127; GFX942-NEXT: s_waitcnt lgkmcnt(0)2128; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32129; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]2130; GFX942-NEXT: v_mov_b32_e32 v3, v42131; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]2132; GFX942-NEXT: s_cbranch_execnz .LBB10_12133; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end2134; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]2135; GFX942-NEXT: s_setpc_b64 s[30:31]2136;2137; GFX11-TRUE16-LABEL: local_atomic_fadd_noret_f16:2138; GFX11-TRUE16: ; %bb.0:2139; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2140; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, -4, v02141; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v02142; GFX11-TRUE16-NEXT: s_mov_b32 s0, 02143; GFX11-TRUE16-NEXT: ds_load_b32 v2, v12144; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff2145; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 24, v02146; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)2147; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v32148; GFX11-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start2149; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=12150; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)2151; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)2152; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v22153; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 02154; GFX11-TRUE16-NEXT: v_add_f16_e32 v4.l, 4.0, v4.l2155; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2156; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v42157; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v42158; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x02159; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v22160; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)2161; GFX11-TRUE16-NEXT: buffer_gl0_inv2162; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v22163; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v42164; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s02165; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2166; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02167; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB10_12168; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end2169; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s02170; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]2171;2172; GFX11-FAKE16-LABEL: local_atomic_fadd_noret_f16:2173; GFX11-FAKE16: ; %bb.0:2174; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2175; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, -4, v02176; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 3, v02177; GFX11-FAKE16-NEXT: s_mov_b32 s0, 02178; GFX11-FAKE16-NEXT: ds_load_b32 v2, v12179; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff2180; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 24, v02181; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)2182; GFX11-FAKE16-NEXT: v_not_b32_e32 v3, v32183; GFX11-FAKE16-NEXT: .LBB10_1: ; %atomicrmw.start2184; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=12185; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)2186; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2187; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v0, v22188; GFX11-FAKE16-NEXT: v_add_f16_e32 v4, 4.0, v42189; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2190; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v42191; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v0, v42192; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2193; GFX11-FAKE16-NEXT: v_and_or_b32 v4, v2, v3, v42194; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x02195; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v22196; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)2197; GFX11-FAKE16-NEXT: buffer_gl0_inv2198; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v22199; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v42200; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s02201; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2202; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02203; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB10_12204; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end2205; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s02206; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]2207;2208; GFX10-LABEL: local_atomic_fadd_noret_f16:2209; GFX10: ; %bb.0:2210; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2211; GFX10-NEXT: v_and_b32_e32 v1, -4, v02212; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v02213; GFX10-NEXT: s_mov_b32 s4, 02214; GFX10-NEXT: ds_read_b32 v2, v12215; GFX10-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff2216; GFX10-NEXT: v_and_b32_e32 v0, 24, v02217; GFX10-NEXT: v_not_b32_e32 v3, v32218; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start2219; GFX10-NEXT: ; =>This Inner Loop Header: Depth=12220; GFX10-NEXT: s_waitcnt lgkmcnt(0)2221; GFX10-NEXT: v_lshrrev_b32_e32 v4, v0, v22222; GFX10-NEXT: v_add_f16_e32 v4, 4.0, v42223; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_02224; GFX10-NEXT: v_and_or_b32 v4, v2, v3, v42225; GFX10-NEXT: s_waitcnt_vscnt null, 0x02226; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v1, v2, v42227; GFX10-NEXT: s_waitcnt lgkmcnt(0)2228; GFX10-NEXT: buffer_gl0_inv2229; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v22230; GFX10-NEXT: v_mov_b32_e32 v2, v42231; GFX10-NEXT: s_or_b32 s4, vcc_lo, s42232; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s42233; GFX10-NEXT: s_cbranch_execnz .LBB10_12234; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end2235; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s42236; GFX10-NEXT: s_setpc_b64 s[30:31]2237;2238; GFX90A-LABEL: local_atomic_fadd_noret_f16:2239; GFX90A: ; %bb.0:2240; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2241; GFX90A-NEXT: v_and_b32_e32 v1, -4, v02242; GFX90A-NEXT: ds_read_b32 v3, v12243; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 3, v02244; GFX90A-NEXT: s_mov_b32 s4, 0xffff2245; GFX90A-NEXT: v_and_b32_e32 v0, 24, v22246; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v2, s42247; GFX90A-NEXT: v_not_b32_e32 v2, v22248; GFX90A-NEXT: s_mov_b64 s[4:5], 02249; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start2250; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12251; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2252; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v0, v32253; GFX90A-NEXT: v_add_f16_e32 v4, 4.0, v42254; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v0, v42255; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v42256; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v42257; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2258; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32259; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2260; GFX90A-NEXT: v_mov_b32_e32 v3, v42261; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]2262; GFX90A-NEXT: s_cbranch_execnz .LBB10_12263; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end2264; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2265; GFX90A-NEXT: s_setpc_b64 s[30:31]2266;2267; GFX908-LABEL: local_atomic_fadd_noret_f16:2268; GFX908: ; %bb.0:2269; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2270; GFX908-NEXT: v_and_b32_e32 v1, -4, v02271; GFX908-NEXT: ds_read_b32 v3, v12272; GFX908-NEXT: v_lshlrev_b32_e32 v2, 3, v02273; GFX908-NEXT: s_mov_b32 s4, 0xffff2274; GFX908-NEXT: v_and_b32_e32 v0, 24, v22275; GFX908-NEXT: v_lshlrev_b32_e64 v2, v2, s42276; GFX908-NEXT: v_not_b32_e32 v2, v22277; GFX908-NEXT: s_mov_b64 s[4:5], 02278; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start2279; GFX908-NEXT: ; =>This Inner Loop Header: Depth=12280; GFX908-NEXT: s_waitcnt lgkmcnt(0)2281; GFX908-NEXT: v_lshrrev_b32_e32 v4, v0, v32282; GFX908-NEXT: v_add_f16_e32 v4, 4.0, v42283; GFX908-NEXT: v_lshlrev_b32_e32 v4, v0, v42284; GFX908-NEXT: v_and_or_b32 v4, v3, v2, v42285; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v42286; GFX908-NEXT: s_waitcnt lgkmcnt(0)2287; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32288; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2289; GFX908-NEXT: v_mov_b32_e32 v3, v42290; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]2291; GFX908-NEXT: s_cbranch_execnz .LBB10_12292; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end2293; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]2294; GFX908-NEXT: s_setpc_b64 s[30:31]2295;2296; GFX8-LABEL: local_atomic_fadd_noret_f16:2297; GFX8: ; %bb.0:2298; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2299; GFX8-NEXT: v_and_b32_e32 v1, -4, v02300; GFX8-NEXT: s_mov_b32 m0, -12301; GFX8-NEXT: ds_read_b32 v3, v12302; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v02303; GFX8-NEXT: s_mov_b32 s4, 0xffff2304; GFX8-NEXT: v_and_b32_e32 v0, 24, v22305; GFX8-NEXT: v_lshlrev_b32_e64 v2, v2, s42306; GFX8-NEXT: v_not_b32_e32 v2, v22307; GFX8-NEXT: s_mov_b64 s[4:5], 02308; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start2309; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12310; GFX8-NEXT: s_waitcnt lgkmcnt(0)2311; GFX8-NEXT: v_lshrrev_b32_e32 v4, v0, v32312; GFX8-NEXT: v_add_f16_e32 v4, 4.0, v42313; GFX8-NEXT: v_and_b32_e32 v5, v3, v22314; GFX8-NEXT: v_lshlrev_b32_e32 v4, v0, v42315; GFX8-NEXT: v_or_b32_e32 v4, v5, v42316; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v42317; GFX8-NEXT: s_waitcnt lgkmcnt(0)2318; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32319; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2320; GFX8-NEXT: v_mov_b32_e32 v3, v42321; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]2322; GFX8-NEXT: s_cbranch_execnz .LBB10_12323; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2324; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]2325; GFX8-NEXT: s_setpc_b64 s[30:31]2326;2327; GFX7-LABEL: local_atomic_fadd_noret_f16:2328; GFX7: ; %bb.0:2329; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2330; GFX7-NEXT: v_and_b32_e32 v1, -4, v02331; GFX7-NEXT: s_mov_b32 m0, -12332; GFX7-NEXT: ds_read_b32 v3, v12333; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v02334; GFX7-NEXT: v_and_b32_e32 v0, 24, v22335; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v22336; GFX7-NEXT: v_not_b32_e32 v2, v22337; GFX7-NEXT: s_mov_b64 s[4:5], 02338; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start2339; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12340; GFX7-NEXT: s_waitcnt lgkmcnt(0)2341; GFX7-NEXT: v_lshrrev_b32_e32 v4, v0, v32342; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v42343; GFX7-NEXT: v_and_b32_e32 v5, v3, v22344; GFX7-NEXT: v_add_f32_e32 v4, 4.0, v42345; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v42346; GFX7-NEXT: v_lshlrev_b32_e32 v4, v0, v42347; GFX7-NEXT: v_or_b32_e32 v4, v5, v42348; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v42349; GFX7-NEXT: s_waitcnt lgkmcnt(0)2350; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32351; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2352; GFX7-NEXT: v_mov_b32_e32 v3, v42353; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]2354; GFX7-NEXT: s_cbranch_execnz .LBB10_12355; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2356; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2357; GFX7-NEXT: s_setpc_b64 s[30:31]2358;2359; GFX6-LABEL: local_atomic_fadd_noret_f16:2360; GFX6: ; %bb.0:2361; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2362; GFX6-NEXT: v_and_b32_e32 v1, -4, v02363; GFX6-NEXT: s_mov_b32 m0, -12364; GFX6-NEXT: ds_read_b32 v3, v12365; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v02366; GFX6-NEXT: v_and_b32_e32 v0, 24, v22367; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v22368; GFX6-NEXT: v_not_b32_e32 v2, v22369; GFX6-NEXT: s_mov_b64 s[4:5], 02370; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start2371; GFX6-NEXT: ; =>This Inner Loop Header: Depth=12372; GFX6-NEXT: s_waitcnt lgkmcnt(0)2373; GFX6-NEXT: v_lshrrev_b32_e32 v4, v0, v32374; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v42375; GFX6-NEXT: v_and_b32_e32 v5, v3, v22376; GFX6-NEXT: v_add_f32_e32 v4, 4.0, v42377; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v42378; GFX6-NEXT: v_lshlrev_b32_e32 v4, v0, v42379; GFX6-NEXT: v_or_b32_e32 v4, v5, v42380; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v42381; GFX6-NEXT: s_waitcnt lgkmcnt(0)2382; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32383; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2384; GFX6-NEXT: v_mov_b32_e32 v3, v42385; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]2386; GFX6-NEXT: s_cbranch_execnz .LBB10_12387; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end2388; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]2389; GFX6-NEXT: s_setpc_b64 s[30:31]2390 %result = atomicrmw fadd ptr addrspace(3) %ptr, half 4.0 seq_cst2391 ret void2392}2393 2394define void @local_atomic_fadd_noret_f16__offset(ptr addrspace(3) %ptr) nounwind {2395; GFX12-TRUE16-LABEL: local_atomic_fadd_noret_f16__offset:2396; GFX12-TRUE16: ; %bb.0:2397; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x02398; GFX12-TRUE16-NEXT: s_wait_expcnt 0x02399; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x02400; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x02401; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x02402; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v02403; GFX12-TRUE16-NEXT: s_mov_b32 s0, 02404; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)2405; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v12406; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 3, v12407; GFX12-TRUE16-NEXT: ds_load_b32 v3, v02408; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v12409; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff2410; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2411; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v22412; GFX12-TRUE16-NEXT: .LBB11_1: ; %atomicrmw.start2413; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=12414; GFX12-TRUE16-NEXT: s_wait_dscnt 0x02415; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v32416; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 02417; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2418; GFX12-TRUE16-NEXT: v_add_f16_e32 v4.l, 4.0, v4.l2419; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v42420; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2421; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v42422; GFX12-TRUE16-NEXT: s_wait_storecnt 0x02423; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v32424; GFX12-TRUE16-NEXT: s_wait_dscnt 0x02425; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE2426; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v32427; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v42428; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)2429; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s02430; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)2431; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02432; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB11_12433; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end2434; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s02435; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x02436; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]2437;2438; GFX12-FAKE16-LABEL: local_atomic_fadd_noret_f16__offset:2439; GFX12-FAKE16: ; %bb.0:2440; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x02441; GFX12-FAKE16-NEXT: s_wait_expcnt 0x02442; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x02443; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x02444; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x02445; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v02446; GFX12-FAKE16-NEXT: s_mov_b32 s0, 02447; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)2448; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v12449; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 3, v12450; GFX12-FAKE16-NEXT: ds_load_b32 v3, v02451; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 3, v12452; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff2453; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2454; GFX12-FAKE16-NEXT: v_not_b32_e32 v2, v22455; GFX12-FAKE16-NEXT: .LBB11_1: ; %atomicrmw.start2456; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=12457; GFX12-FAKE16-NEXT: s_wait_dscnt 0x02458; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v1, v32459; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2460; GFX12-FAKE16-NEXT: v_add_f16_e32 v4, 4.0, v42461; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v42462; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2463; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v1, v42464; GFX12-FAKE16-NEXT: v_and_or_b32 v4, v3, v2, v42465; GFX12-FAKE16-NEXT: s_wait_storecnt 0x02466; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v32467; GFX12-FAKE16-NEXT: s_wait_dscnt 0x02468; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE2469; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v32470; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v42471; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)2472; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s02473; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)2474; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02475; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB11_12476; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end2477; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s02478; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x02479; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]2480;2481; GFX942-LABEL: local_atomic_fadd_noret_f16__offset:2482; GFX942: ; %bb.0:2483; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2484; GFX942-NEXT: v_add_u32_e32 v1, 0xfffe, v02485; GFX942-NEXT: v_and_b32_e32 v0, -4, v12486; GFX942-NEXT: ds_read_b32 v3, v02487; GFX942-NEXT: v_and_b32_e32 v1, 3, v12488; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v12489; GFX942-NEXT: s_mov_b32 s0, 0xffff2490; GFX942-NEXT: v_lshlrev_b32_e64 v2, v1, s02491; GFX942-NEXT: v_not_b32_e32 v2, v22492; GFX942-NEXT: s_mov_b64 s[0:1], 02493; GFX942-NEXT: .LBB11_1: ; %atomicrmw.start2494; GFX942-NEXT: ; =>This Inner Loop Header: Depth=12495; GFX942-NEXT: s_waitcnt lgkmcnt(0)2496; GFX942-NEXT: v_lshrrev_b32_e32 v4, v1, v32497; GFX942-NEXT: v_add_f16_e32 v4, 4.0, v42498; GFX942-NEXT: v_lshlrev_b32_e32 v4, v1, v42499; GFX942-NEXT: v_and_or_b32 v4, v3, v2, v42500; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v42501; GFX942-NEXT: s_waitcnt lgkmcnt(0)2502; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32503; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]2504; GFX942-NEXT: v_mov_b32_e32 v3, v42505; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]2506; GFX942-NEXT: s_cbranch_execnz .LBB11_12507; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end2508; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]2509; GFX942-NEXT: s_setpc_b64 s[30:31]2510;2511; GFX11-TRUE16-LABEL: local_atomic_fadd_noret_f16__offset:2512; GFX11-TRUE16: ; %bb.0:2513; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2514; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v02515; GFX11-TRUE16-NEXT: s_mov_b32 s0, 02516; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)2517; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v12518; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 3, v12519; GFX11-TRUE16-NEXT: ds_load_b32 v3, v02520; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v12521; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff2522; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2523; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v22524; GFX11-TRUE16-NEXT: .LBB11_1: ; %atomicrmw.start2525; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=12526; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)2527; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v32528; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 02529; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2530; GFX11-TRUE16-NEXT: v_add_f16_e32 v4.l, 4.0, v4.l2531; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v42532; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2533; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v42534; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x02535; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v32536; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)2537; GFX11-TRUE16-NEXT: buffer_gl0_inv2538; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v32539; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v42540; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s02541; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2542; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02543; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB11_12544; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end2545; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s02546; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]2547;2548; GFX11-FAKE16-LABEL: local_atomic_fadd_noret_f16__offset:2549; GFX11-FAKE16: ; %bb.0:2550; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2551; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v02552; GFX11-FAKE16-NEXT: s_mov_b32 s0, 02553; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)2554; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v12555; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 3, v12556; GFX11-FAKE16-NEXT: ds_load_b32 v3, v02557; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 3, v12558; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff2559; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2560; GFX11-FAKE16-NEXT: v_not_b32_e32 v2, v22561; GFX11-FAKE16-NEXT: .LBB11_1: ; %atomicrmw.start2562; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=12563; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)2564; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v1, v32565; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2566; GFX11-FAKE16-NEXT: v_add_f16_e32 v4, 4.0, v42567; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v42568; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2569; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v1, v42570; GFX11-FAKE16-NEXT: v_and_or_b32 v4, v3, v2, v42571; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x02572; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v32573; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)2574; GFX11-FAKE16-NEXT: buffer_gl0_inv2575; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v32576; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v42577; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s02578; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2579; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02580; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB11_12581; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end2582; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s02583; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]2584;2585; GFX10-LABEL: local_atomic_fadd_noret_f16__offset:2586; GFX10: ; %bb.0:2587; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2588; GFX10-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v02589; GFX10-NEXT: s_mov_b32 s4, 02590; GFX10-NEXT: v_and_b32_e32 v0, -4, v12591; GFX10-NEXT: v_and_b32_e32 v1, 3, v12592; GFX10-NEXT: ds_read_b32 v3, v02593; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v12594; GFX10-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff2595; GFX10-NEXT: v_not_b32_e32 v2, v22596; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start2597; GFX10-NEXT: ; =>This Inner Loop Header: Depth=12598; GFX10-NEXT: s_waitcnt lgkmcnt(0)2599; GFX10-NEXT: v_lshrrev_b32_e32 v4, v1, v32600; GFX10-NEXT: v_add_f16_e32 v4, 4.0, v42601; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_02602; GFX10-NEXT: v_and_or_b32 v4, v3, v2, v42603; GFX10-NEXT: s_waitcnt_vscnt null, 0x02604; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v42605; GFX10-NEXT: s_waitcnt lgkmcnt(0)2606; GFX10-NEXT: buffer_gl0_inv2607; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v32608; GFX10-NEXT: v_mov_b32_e32 v3, v42609; GFX10-NEXT: s_or_b32 s4, vcc_lo, s42610; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s42611; GFX10-NEXT: s_cbranch_execnz .LBB11_12612; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end2613; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s42614; GFX10-NEXT: s_setpc_b64 s[30:31]2615;2616; GFX90A-LABEL: local_atomic_fadd_noret_f16__offset:2617; GFX90A: ; %bb.0:2618; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2619; GFX90A-NEXT: v_add_u32_e32 v1, 0xfffe, v02620; GFX90A-NEXT: v_and_b32_e32 v0, -4, v12621; GFX90A-NEXT: ds_read_b32 v3, v02622; GFX90A-NEXT: v_and_b32_e32 v1, 3, v12623; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 3, v12624; GFX90A-NEXT: s_mov_b32 s4, 0xffff2625; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v1, s42626; GFX90A-NEXT: v_not_b32_e32 v2, v22627; GFX90A-NEXT: s_mov_b64 s[4:5], 02628; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start2629; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12630; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2631; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v1, v32632; GFX90A-NEXT: v_add_f16_e32 v4, 4.0, v42633; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v1, v42634; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v42635; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v42636; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2637; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32638; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2639; GFX90A-NEXT: v_mov_b32_e32 v3, v42640; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]2641; GFX90A-NEXT: s_cbranch_execnz .LBB11_12642; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end2643; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2644; GFX90A-NEXT: s_setpc_b64 s[30:31]2645;2646; GFX908-LABEL: local_atomic_fadd_noret_f16__offset:2647; GFX908: ; %bb.0:2648; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2649; GFX908-NEXT: v_add_u32_e32 v1, 0xfffe, v02650; GFX908-NEXT: v_and_b32_e32 v0, -4, v12651; GFX908-NEXT: ds_read_b32 v3, v02652; GFX908-NEXT: v_and_b32_e32 v1, 3, v12653; GFX908-NEXT: v_lshlrev_b32_e32 v1, 3, v12654; GFX908-NEXT: s_mov_b32 s4, 0xffff2655; GFX908-NEXT: v_lshlrev_b32_e64 v2, v1, s42656; GFX908-NEXT: v_not_b32_e32 v2, v22657; GFX908-NEXT: s_mov_b64 s[4:5], 02658; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start2659; GFX908-NEXT: ; =>This Inner Loop Header: Depth=12660; GFX908-NEXT: s_waitcnt lgkmcnt(0)2661; GFX908-NEXT: v_lshrrev_b32_e32 v4, v1, v32662; GFX908-NEXT: v_add_f16_e32 v4, 4.0, v42663; GFX908-NEXT: v_lshlrev_b32_e32 v4, v1, v42664; GFX908-NEXT: v_and_or_b32 v4, v3, v2, v42665; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v42666; GFX908-NEXT: s_waitcnt lgkmcnt(0)2667; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32668; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2669; GFX908-NEXT: v_mov_b32_e32 v3, v42670; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]2671; GFX908-NEXT: s_cbranch_execnz .LBB11_12672; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end2673; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]2674; GFX908-NEXT: s_setpc_b64 s[30:31]2675;2676; GFX8-LABEL: local_atomic_fadd_noret_f16__offset:2677; GFX8: ; %bb.0:2678; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2679; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0xfffe, v02680; GFX8-NEXT: v_and_b32_e32 v0, -4, v12681; GFX8-NEXT: s_mov_b32 m0, -12682; GFX8-NEXT: ds_read_b32 v3, v02683; GFX8-NEXT: v_and_b32_e32 v1, 3, v12684; GFX8-NEXT: v_lshlrev_b32_e32 v1, 3, v12685; GFX8-NEXT: s_mov_b32 s4, 0xffff2686; GFX8-NEXT: v_lshlrev_b32_e64 v2, v1, s42687; GFX8-NEXT: v_not_b32_e32 v2, v22688; GFX8-NEXT: s_mov_b64 s[4:5], 02689; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start2690; GFX8-NEXT: ; =>This Inner Loop Header: Depth=12691; GFX8-NEXT: s_waitcnt lgkmcnt(0)2692; GFX8-NEXT: v_lshrrev_b32_e32 v4, v1, v32693; GFX8-NEXT: v_add_f16_e32 v4, 4.0, v42694; GFX8-NEXT: v_and_b32_e32 v5, v3, v22695; GFX8-NEXT: v_lshlrev_b32_e32 v4, v1, v42696; GFX8-NEXT: v_or_b32_e32 v4, v5, v42697; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v42698; GFX8-NEXT: s_waitcnt lgkmcnt(0)2699; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32700; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2701; GFX8-NEXT: v_mov_b32_e32 v3, v42702; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]2703; GFX8-NEXT: s_cbranch_execnz .LBB11_12704; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end2705; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]2706; GFX8-NEXT: s_setpc_b64 s[30:31]2707;2708; GFX7-LABEL: local_atomic_fadd_noret_f16__offset:2709; GFX7: ; %bb.0:2710; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2711; GFX7-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v02712; GFX7-NEXT: v_and_b32_e32 v0, -4, v12713; GFX7-NEXT: s_mov_b32 m0, -12714; GFX7-NEXT: ds_read_b32 v3, v02715; GFX7-NEXT: v_and_b32_e32 v1, 3, v12716; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v12717; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v12718; GFX7-NEXT: v_not_b32_e32 v2, v22719; GFX7-NEXT: s_mov_b64 s[4:5], 02720; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start2721; GFX7-NEXT: ; =>This Inner Loop Header: Depth=12722; GFX7-NEXT: s_waitcnt lgkmcnt(0)2723; GFX7-NEXT: v_lshrrev_b32_e32 v4, v1, v32724; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v42725; GFX7-NEXT: v_and_b32_e32 v5, v3, v22726; GFX7-NEXT: v_add_f32_e32 v4, 4.0, v42727; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v42728; GFX7-NEXT: v_lshlrev_b32_e32 v4, v1, v42729; GFX7-NEXT: v_or_b32_e32 v4, v5, v42730; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v42731; GFX7-NEXT: s_waitcnt lgkmcnt(0)2732; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32733; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2734; GFX7-NEXT: v_mov_b32_e32 v3, v42735; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]2736; GFX7-NEXT: s_cbranch_execnz .LBB11_12737; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end2738; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]2739; GFX7-NEXT: s_setpc_b64 s[30:31]2740;2741; GFX6-LABEL: local_atomic_fadd_noret_f16__offset:2742; GFX6: ; %bb.0:2743; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2744; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v02745; GFX6-NEXT: v_and_b32_e32 v0, -4, v12746; GFX6-NEXT: s_mov_b32 m0, -12747; GFX6-NEXT: ds_read_b32 v3, v02748; GFX6-NEXT: v_and_b32_e32 v1, 3, v12749; GFX6-NEXT: v_lshlrev_b32_e32 v1, 3, v12750; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v12751; GFX6-NEXT: v_not_b32_e32 v2, v22752; GFX6-NEXT: s_mov_b64 s[4:5], 02753; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start2754; GFX6-NEXT: ; =>This Inner Loop Header: Depth=12755; GFX6-NEXT: s_waitcnt lgkmcnt(0)2756; GFX6-NEXT: v_lshrrev_b32_e32 v4, v1, v32757; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v42758; GFX6-NEXT: v_and_b32_e32 v5, v3, v22759; GFX6-NEXT: v_add_f32_e32 v4, 4.0, v42760; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v42761; GFX6-NEXT: v_lshlrev_b32_e32 v4, v1, v42762; GFX6-NEXT: v_or_b32_e32 v4, v5, v42763; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v42764; GFX6-NEXT: s_waitcnt lgkmcnt(0)2765; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v32766; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2767; GFX6-NEXT: v_mov_b32_e32 v3, v42768; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]2769; GFX6-NEXT: s_cbranch_execnz .LBB11_12770; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end2771; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]2772; GFX6-NEXT: s_setpc_b64 s[30:31]2773 %gep = getelementptr half, ptr addrspace(3) %ptr, i32 327672774 %unused = atomicrmw fadd ptr addrspace(3) %gep, half 4.0 seq_cst2775 ret void2776}2777 2778define half @local_atomic_fadd_ret_f16__offset__align4(ptr addrspace(3) %ptr) nounwind {2779; GFX12-TRUE16-LABEL: local_atomic_fadd_ret_f16__offset__align4:2780; GFX12-TRUE16: ; %bb.0:2781; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x02782; GFX12-TRUE16-NEXT: s_wait_expcnt 0x02783; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x02784; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x02785; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x02786; GFX12-TRUE16-NEXT: ds_load_b32 v1, v0 offset:655342787; GFX12-TRUE16-NEXT: s_mov_b32 s0, 02788; GFX12-TRUE16-NEXT: .LBB12_1: ; %atomicrmw.start2789; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=12790; GFX12-TRUE16-NEXT: s_wait_dscnt 0x02791; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v12792; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 02793; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2794; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, 4.0, v2.l2795; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v12796; GFX12-TRUE16-NEXT: s_wait_storecnt 0x02797; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:655342798; GFX12-TRUE16-NEXT: s_wait_dscnt 0x02799; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE2800; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v22801; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)2802; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s02803; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)2804; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02805; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_12806; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end2807; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s02808; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l2809; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x02810; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]2811;2812; GFX12-FAKE16-LABEL: local_atomic_fadd_ret_f16__offset__align4:2813; GFX12-FAKE16: ; %bb.0:2814; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x02815; GFX12-FAKE16-NEXT: s_wait_expcnt 0x02816; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x02817; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x02818; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x02819; GFX12-FAKE16-NEXT: ds_load_b32 v1, v0 offset:655342820; GFX12-FAKE16-NEXT: s_mov_b32 s0, 02821; GFX12-FAKE16-NEXT: .LBB12_1: ; %atomicrmw.start2822; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=12823; GFX12-FAKE16-NEXT: s_wait_dscnt 0x02824; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v12825; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2826; GFX12-FAKE16-NEXT: v_add_f16_e32 v1, 4.0, v22827; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v12828; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2829; GFX12-FAKE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v12830; GFX12-FAKE16-NEXT: s_wait_storecnt 0x02831; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:655342832; GFX12-FAKE16-NEXT: s_wait_dscnt 0x02833; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE2834; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v22835; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)2836; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s02837; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)2838; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02839; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_12840; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end2841; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s02842; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v12843; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x02844; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]2845;2846; GFX942-LABEL: local_atomic_fadd_ret_f16__offset__align4:2847; GFX942: ; %bb.0:2848; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2849; GFX942-NEXT: ds_read_b32 v2, v0 offset:655342850; GFX942-NEXT: s_mov_b64 s[0:1], 02851; GFX942-NEXT: s_mov_b32 s2, 0xffff00002852; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start2853; GFX942-NEXT: ; =>This Inner Loop Header: Depth=12854; GFX942-NEXT: s_waitcnt lgkmcnt(0)2855; GFX942-NEXT: v_add_f16_e32 v1, 4.0, v22856; GFX942-NEXT: v_and_or_b32 v1, v2, s2, v12857; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655342858; GFX942-NEXT: s_waitcnt lgkmcnt(0)2859; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v22860; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]2861; GFX942-NEXT: v_mov_b32_e32 v2, v12862; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]2863; GFX942-NEXT: s_cbranch_execnz .LBB12_12864; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end2865; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]2866; GFX942-NEXT: v_mov_b32_e32 v0, v12867; GFX942-NEXT: s_setpc_b64 s[30:31]2868;2869; GFX11-TRUE16-LABEL: local_atomic_fadd_ret_f16__offset__align4:2870; GFX11-TRUE16: ; %bb.0:2871; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2872; GFX11-TRUE16-NEXT: ds_load_b32 v1, v0 offset:655342873; GFX11-TRUE16-NEXT: s_mov_b32 s0, 02874; GFX11-TRUE16-NEXT: .LBB12_1: ; %atomicrmw.start2875; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=12876; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)2877; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v12878; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 02879; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)2880; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, 4.0, v2.l2881; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v12882; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x02883; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:655342884; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)2885; GFX11-TRUE16-NEXT: buffer_gl0_inv2886; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v22887; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s02888; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2889; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02890; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_12891; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end2892; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s02893; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l2894; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]2895;2896; GFX11-FAKE16-LABEL: local_atomic_fadd_ret_f16__offset__align4:2897; GFX11-FAKE16: ; %bb.0:2898; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2899; GFX11-FAKE16-NEXT: ds_load_b32 v1, v0 offset:655342900; GFX11-FAKE16-NEXT: s_mov_b32 s0, 02901; GFX11-FAKE16-NEXT: .LBB12_1: ; %atomicrmw.start2902; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=12903; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)2904; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v12905; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)2906; GFX11-FAKE16-NEXT: v_add_f16_e32 v1, 4.0, v22907; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v12908; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)2909; GFX11-FAKE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v12910; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x02911; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:655342912; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)2913; GFX11-FAKE16-NEXT: buffer_gl0_inv2914; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v22915; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s02916; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)2917; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s02918; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_12919; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end2920; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s02921; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v12922; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]2923;2924; GFX10-LABEL: local_atomic_fadd_ret_f16__offset__align4:2925; GFX10: ; %bb.0:2926; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2927; GFX10-NEXT: ds_read_b32 v1, v0 offset:655342928; GFX10-NEXT: s_mov_b32 s4, 02929; GFX10-NEXT: .LBB12_1: ; %atomicrmw.start2930; GFX10-NEXT: ; =>This Inner Loop Header: Depth=12931; GFX10-NEXT: s_waitcnt lgkmcnt(0)2932; GFX10-NEXT: v_mov_b32_e32 v2, v12933; GFX10-NEXT: v_add_f16_e32 v1, 4.0, v22934; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v12935; GFX10-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v12936; GFX10-NEXT: s_waitcnt_vscnt null, 0x02937; GFX10-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655342938; GFX10-NEXT: s_waitcnt lgkmcnt(0)2939; GFX10-NEXT: buffer_gl0_inv2940; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v22941; GFX10-NEXT: s_or_b32 s4, vcc_lo, s42942; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s42943; GFX10-NEXT: s_cbranch_execnz .LBB12_12944; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end2945; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s42946; GFX10-NEXT: v_mov_b32_e32 v0, v12947; GFX10-NEXT: s_setpc_b64 s[30:31]2948;2949; GFX90A-LABEL: local_atomic_fadd_ret_f16__offset__align4:2950; GFX90A: ; %bb.0:2951; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2952; GFX90A-NEXT: ds_read_b32 v2, v0 offset:655342953; GFX90A-NEXT: s_mov_b64 s[4:5], 02954; GFX90A-NEXT: s_mov_b32 s6, 0xffff00002955; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start2956; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=12957; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2958; GFX90A-NEXT: v_add_f16_e32 v1, 4.0, v22959; GFX90A-NEXT: v_and_or_b32 v1, v2, s6, v12960; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655342961; GFX90A-NEXT: s_waitcnt lgkmcnt(0)2962; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v22963; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2964; GFX90A-NEXT: v_mov_b32_e32 v2, v12965; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]2966; GFX90A-NEXT: s_cbranch_execnz .LBB12_12967; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end2968; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]2969; GFX90A-NEXT: v_mov_b32_e32 v0, v12970; GFX90A-NEXT: s_setpc_b64 s[30:31]2971;2972; GFX908-LABEL: local_atomic_fadd_ret_f16__offset__align4:2973; GFX908: ; %bb.0:2974; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2975; GFX908-NEXT: ds_read_b32 v1, v0 offset:655342976; GFX908-NEXT: s_mov_b64 s[4:5], 02977; GFX908-NEXT: s_mov_b32 s6, 0xffff00002978; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start2979; GFX908-NEXT: ; =>This Inner Loop Header: Depth=12980; GFX908-NEXT: s_waitcnt lgkmcnt(0)2981; GFX908-NEXT: v_mov_b32_e32 v2, v12982; GFX908-NEXT: v_add_f16_e32 v1, 4.0, v22983; GFX908-NEXT: v_and_or_b32 v1, v2, s6, v12984; GFX908-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655342985; GFX908-NEXT: s_waitcnt lgkmcnt(0)2986; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v22987; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]2988; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]2989; GFX908-NEXT: s_cbranch_execnz .LBB12_12990; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end2991; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]2992; GFX908-NEXT: v_mov_b32_e32 v0, v12993; GFX908-NEXT: s_setpc_b64 s[30:31]2994;2995; GFX8-LABEL: local_atomic_fadd_ret_f16__offset__align4:2996; GFX8: ; %bb.0:2997; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)2998; GFX8-NEXT: s_mov_b32 m0, -12999; GFX8-NEXT: ds_read_b32 v1, v0 offset:655343000; GFX8-NEXT: s_mov_b64 s[4:5], 03001; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start3002; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13003; GFX8-NEXT: s_waitcnt lgkmcnt(0)3004; GFX8-NEXT: v_mov_b32_e32 v2, v13005; GFX8-NEXT: v_add_f16_e32 v1, 4.0, v23006; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v23007; GFX8-NEXT: v_or_b32_e32 v1, v3, v13008; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655343009; GFX8-NEXT: s_waitcnt lgkmcnt(0)3010; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v23011; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3012; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3013; GFX8-NEXT: s_cbranch_execnz .LBB12_13014; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3015; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3016; GFX8-NEXT: v_mov_b32_e32 v0, v13017; GFX8-NEXT: s_setpc_b64 s[30:31]3018;3019; GFX7-LABEL: local_atomic_fadd_ret_f16__offset__align4:3020; GFX7: ; %bb.0:3021; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3022; GFX7-NEXT: s_mov_b32 m0, -13023; GFX7-NEXT: ds_read_b32 v1, v0 offset:655343024; GFX7-NEXT: s_mov_b64 s[4:5], 03025; GFX7-NEXT: .LBB12_1: ; %atomicrmw.start3026; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13027; GFX7-NEXT: s_waitcnt lgkmcnt(0)3028; GFX7-NEXT: v_mov_b32_e32 v2, v13029; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v23030; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v23031; GFX7-NEXT: v_add_f32_e32 v1, 4.0, v13032; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v13033; GFX7-NEXT: v_or_b32_e32 v1, v3, v13034; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655343035; GFX7-NEXT: s_waitcnt lgkmcnt(0)3036; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v23037; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3038; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3039; GFX7-NEXT: s_cbranch_execnz .LBB12_13040; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3041; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3042; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v13043; GFX7-NEXT: s_setpc_b64 s[30:31]3044;3045; GFX6-LABEL: local_atomic_fadd_ret_f16__offset__align4:3046; GFX6: ; %bb.0:3047; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3048; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v03049; GFX6-NEXT: s_mov_b32 m0, -13050; GFX6-NEXT: ds_read_b32 v1, v03051; GFX6-NEXT: s_mov_b64 s[4:5], 03052; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start3053; GFX6-NEXT: ; =>This Inner Loop Header: Depth=13054; GFX6-NEXT: s_waitcnt lgkmcnt(0)3055; GFX6-NEXT: v_mov_b32_e32 v2, v13056; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v23057; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v23058; GFX6-NEXT: v_add_f32_e32 v1, 4.0, v13059; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v13060; GFX6-NEXT: v_or_b32_e32 v1, v3, v13061; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v13062; GFX6-NEXT: s_waitcnt lgkmcnt(0)3063; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v23064; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3065; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]3066; GFX6-NEXT: s_cbranch_execnz .LBB12_13067; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end3068; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]3069; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v13070; GFX6-NEXT: s_setpc_b64 s[30:31]3071 %gep = getelementptr half, ptr addrspace(3) %ptr, i32 327673072 %result = atomicrmw fadd ptr addrspace(3) %gep, half 4.0 seq_cst, align 43073 ret half %result3074}3075 3076define void @local_atomic_fadd_noret_f16__offset__align4(ptr addrspace(3) %ptr) nounwind {3077; GFX12-TRUE16-LABEL: local_atomic_fadd_noret_f16__offset__align4:3078; GFX12-TRUE16: ; %bb.0:3079; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x03080; GFX12-TRUE16-NEXT: s_wait_expcnt 0x03081; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x03082; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x03083; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x03084; GFX12-TRUE16-NEXT: ds_load_b32 v1, v0 offset:655343085; GFX12-TRUE16-NEXT: s_mov_b32 s0, 03086; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start3087; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=13088; GFX12-TRUE16-NEXT: s_wait_dscnt 0x03089; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.l, 4.0, v1.l3090; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 03091; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3092; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v23093; GFX12-TRUE16-NEXT: s_wait_storecnt 0x03094; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:655343095; GFX12-TRUE16-NEXT: s_wait_dscnt 0x03096; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE3097; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v13098; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v23099; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3100; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s03101; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3102; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s03103; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_13104; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end3105; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s03106; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x03107; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]3108;3109; GFX12-FAKE16-LABEL: local_atomic_fadd_noret_f16__offset__align4:3110; GFX12-FAKE16: ; %bb.0:3111; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x03112; GFX12-FAKE16-NEXT: s_wait_expcnt 0x03113; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x03114; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x03115; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x03116; GFX12-FAKE16-NEXT: ds_load_b32 v1, v0 offset:655343117; GFX12-FAKE16-NEXT: s_mov_b32 s0, 03118; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start3119; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=13120; GFX12-FAKE16-NEXT: s_wait_dscnt 0x03121; GFX12-FAKE16-NEXT: v_add_f16_e32 v2, 4.0, v13122; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3123; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v23124; GFX12-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v23125; GFX12-FAKE16-NEXT: s_wait_storecnt 0x03126; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:655343127; GFX12-FAKE16-NEXT: s_wait_dscnt 0x03128; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE3129; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v13130; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v23131; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3132; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s03133; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3134; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s03135; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_13136; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end3137; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s03138; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x03139; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]3140;3141; GFX942-LABEL: local_atomic_fadd_noret_f16__offset__align4:3142; GFX942: ; %bb.0:3143; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3144; GFX942-NEXT: ds_read_b32 v1, v0 offset:655343145; GFX942-NEXT: s_mov_b64 s[0:1], 03146; GFX942-NEXT: s_mov_b32 s2, 0xffff00003147; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start3148; GFX942-NEXT: ; =>This Inner Loop Header: Depth=13149; GFX942-NEXT: s_waitcnt lgkmcnt(0)3150; GFX942-NEXT: v_add_f16_e32 v2, 4.0, v13151; GFX942-NEXT: v_and_or_b32 v2, v1, s2, v23152; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655343153; GFX942-NEXT: s_waitcnt lgkmcnt(0)3154; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v13155; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]3156; GFX942-NEXT: v_mov_b32_e32 v1, v23157; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]3158; GFX942-NEXT: s_cbranch_execnz .LBB13_13159; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end3160; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]3161; GFX942-NEXT: s_setpc_b64 s[30:31]3162;3163; GFX11-TRUE16-LABEL: local_atomic_fadd_noret_f16__offset__align4:3164; GFX11-TRUE16: ; %bb.0:3165; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3166; GFX11-TRUE16-NEXT: ds_load_b32 v1, v0 offset:655343167; GFX11-TRUE16-NEXT: s_mov_b32 s0, 03168; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start3169; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=13170; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)3171; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.l, 4.0, v1.l3172; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 03173; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3174; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v23175; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x03176; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:655343177; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)3178; GFX11-TRUE16-NEXT: buffer_gl0_inv3179; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v13180; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v23181; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s03182; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3183; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s03184; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_13185; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end3186; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s03187; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]3188;3189; GFX11-FAKE16-LABEL: local_atomic_fadd_noret_f16__offset__align4:3190; GFX11-FAKE16: ; %bb.0:3191; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3192; GFX11-FAKE16-NEXT: ds_load_b32 v1, v0 offset:655343193; GFX11-FAKE16-NEXT: s_mov_b32 s0, 03194; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start3195; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=13196; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)3197; GFX11-FAKE16-NEXT: v_add_f16_e32 v2, 4.0, v13198; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3199; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v23200; GFX11-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v23201; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x03202; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:655343203; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)3204; GFX11-FAKE16-NEXT: buffer_gl0_inv3205; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v13206; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v23207; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s03208; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3209; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s03210; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_13211; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end3212; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s03213; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]3214;3215; GFX10-LABEL: local_atomic_fadd_noret_f16__offset__align4:3216; GFX10: ; %bb.0:3217; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3218; GFX10-NEXT: ds_read_b32 v1, v0 offset:655343219; GFX10-NEXT: s_mov_b32 s4, 03220; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start3221; GFX10-NEXT: ; =>This Inner Loop Header: Depth=13222; GFX10-NEXT: s_waitcnt lgkmcnt(0)3223; GFX10-NEXT: v_add_f16_e32 v2, 4.0, v13224; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v23225; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v23226; GFX10-NEXT: s_waitcnt_vscnt null, 0x03227; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655343228; GFX10-NEXT: s_waitcnt lgkmcnt(0)3229; GFX10-NEXT: buffer_gl0_inv3230; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v13231; GFX10-NEXT: v_mov_b32_e32 v1, v23232; GFX10-NEXT: s_or_b32 s4, vcc_lo, s43233; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s43234; GFX10-NEXT: s_cbranch_execnz .LBB13_13235; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end3236; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s43237; GFX10-NEXT: s_setpc_b64 s[30:31]3238;3239; GFX90A-LABEL: local_atomic_fadd_noret_f16__offset__align4:3240; GFX90A: ; %bb.0:3241; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3242; GFX90A-NEXT: ds_read_b32 v1, v0 offset:655343243; GFX90A-NEXT: s_mov_b64 s[4:5], 03244; GFX90A-NEXT: s_mov_b32 s6, 0xffff00003245; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start3246; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=13247; GFX90A-NEXT: s_waitcnt lgkmcnt(0)3248; GFX90A-NEXT: v_add_f16_e32 v2, 4.0, v13249; GFX90A-NEXT: v_and_or_b32 v2, v1, s6, v23250; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655343251; GFX90A-NEXT: s_waitcnt lgkmcnt(0)3252; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v13253; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3254; GFX90A-NEXT: v_mov_b32_e32 v1, v23255; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]3256; GFX90A-NEXT: s_cbranch_execnz .LBB13_13257; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end3258; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3259; GFX90A-NEXT: s_setpc_b64 s[30:31]3260;3261; GFX908-LABEL: local_atomic_fadd_noret_f16__offset__align4:3262; GFX908: ; %bb.0:3263; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3264; GFX908-NEXT: ds_read_b32 v1, v0 offset:655343265; GFX908-NEXT: s_mov_b64 s[4:5], 03266; GFX908-NEXT: s_mov_b32 s6, 0xffff00003267; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start3268; GFX908-NEXT: ; =>This Inner Loop Header: Depth=13269; GFX908-NEXT: s_waitcnt lgkmcnt(0)3270; GFX908-NEXT: v_add_f16_e32 v2, 4.0, v13271; GFX908-NEXT: v_and_or_b32 v2, v1, s6, v23272; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655343273; GFX908-NEXT: s_waitcnt lgkmcnt(0)3274; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v13275; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3276; GFX908-NEXT: v_mov_b32_e32 v1, v23277; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]3278; GFX908-NEXT: s_cbranch_execnz .LBB13_13279; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end3280; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]3281; GFX908-NEXT: s_setpc_b64 s[30:31]3282;3283; GFX8-LABEL: local_atomic_fadd_noret_f16__offset__align4:3284; GFX8: ; %bb.0:3285; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3286; GFX8-NEXT: s_mov_b32 m0, -13287; GFX8-NEXT: ds_read_b32 v1, v0 offset:655343288; GFX8-NEXT: s_mov_b64 s[4:5], 03289; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start3290; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13291; GFX8-NEXT: s_waitcnt lgkmcnt(0)3292; GFX8-NEXT: v_add_f16_e32 v2, 4.0, v13293; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v13294; GFX8-NEXT: v_or_b32_e32 v2, v3, v23295; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655343296; GFX8-NEXT: s_waitcnt lgkmcnt(0)3297; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v13298; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3299; GFX8-NEXT: v_mov_b32_e32 v1, v23300; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3301; GFX8-NEXT: s_cbranch_execnz .LBB13_13302; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3303; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3304; GFX8-NEXT: s_setpc_b64 s[30:31]3305;3306; GFX7-LABEL: local_atomic_fadd_noret_f16__offset__align4:3307; GFX7: ; %bb.0:3308; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3309; GFX7-NEXT: s_mov_b32 m0, -13310; GFX7-NEXT: ds_read_b32 v1, v0 offset:655343311; GFX7-NEXT: s_mov_b64 s[4:5], 03312; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start3313; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13314; GFX7-NEXT: s_waitcnt lgkmcnt(0)3315; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v13316; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v13317; GFX7-NEXT: v_add_f32_e32 v2, 4.0, v23318; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v23319; GFX7-NEXT: v_or_b32_e32 v2, v3, v23320; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655343321; GFX7-NEXT: s_waitcnt lgkmcnt(0)3322; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v13323; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3324; GFX7-NEXT: v_mov_b32_e32 v1, v23325; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3326; GFX7-NEXT: s_cbranch_execnz .LBB13_13327; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3328; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3329; GFX7-NEXT: s_setpc_b64 s[30:31]3330;3331; GFX6-LABEL: local_atomic_fadd_noret_f16__offset__align4:3332; GFX6: ; %bb.0:3333; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3334; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v03335; GFX6-NEXT: s_mov_b32 m0, -13336; GFX6-NEXT: ds_read_b32 v1, v03337; GFX6-NEXT: s_mov_b64 s[4:5], 03338; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start3339; GFX6-NEXT: ; =>This Inner Loop Header: Depth=13340; GFX6-NEXT: s_waitcnt lgkmcnt(0)3341; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v13342; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v13343; GFX6-NEXT: v_add_f32_e32 v2, 4.0, v23344; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v23345; GFX6-NEXT: v_or_b32_e32 v2, v3, v23346; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v23347; GFX6-NEXT: s_waitcnt lgkmcnt(0)3348; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v13349; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3350; GFX6-NEXT: v_mov_b32_e32 v1, v23351; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]3352; GFX6-NEXT: s_cbranch_execnz .LBB13_13353; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end3354; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]3355; GFX6-NEXT: s_setpc_b64 s[30:31]3356 %gep = getelementptr half, ptr addrspace(3) %ptr, i32 327673357 %unused = atomicrmw fadd ptr addrspace(3) %gep, half 4.0 seq_cst, align 43358 ret void3359}3360 3361; --------------------------------------------------------------------3362; bfloat3363; --------------------------------------------------------------------3364 3365define bfloat @local_atomic_fadd_ret_bf16(ptr addrspace(3) %ptr) nounwind {3366; GFX12-TRUE16-LABEL: local_atomic_fadd_ret_bf16:3367; GFX12-TRUE16: ; %bb.0:3368; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x03369; GFX12-TRUE16-NEXT: s_wait_expcnt 0x03370; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x03371; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x03372; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x03373; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, -4, v03374; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v03375; GFX12-TRUE16-NEXT: s_mov_b32 s0, 03376; GFX12-TRUE16-NEXT: ds_load_b32 v3, v13377; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff3378; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 24, v03379; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)3380; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v23381; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start3382; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=13383; GFX12-TRUE16-NEXT: s_wait_dscnt 0x03384; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v33385; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3386; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v43387; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v33388; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3389; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, 4.0, v33390; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 13391; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v33392; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v33393; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)3394; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff3395; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)3396; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo3397; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 03398; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3399; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h3400; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v53401; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3402; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v33403; GFX12-TRUE16-NEXT: s_wait_storecnt 0x03404; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v43405; GFX12-TRUE16-NEXT: s_wait_dscnt 0x03406; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE3407; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v43408; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3409; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s03410; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3411; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s03412; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_13413; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end3414; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s03415; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v33416; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x03417; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]3418;3419; GFX12-FAKE16-LABEL: local_atomic_fadd_ret_bf16:3420; GFX12-FAKE16: ; %bb.0:3421; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x03422; GFX12-FAKE16-NEXT: s_wait_expcnt 0x03423; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x03424; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x03425; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x03426; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, -4, v03427; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 3, v03428; GFX12-FAKE16-NEXT: s_mov_b32 s0, 03429; GFX12-FAKE16-NEXT: ds_load_b32 v3, v13430; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff3431; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 24, v03432; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)3433; GFX12-FAKE16-NEXT: v_not_b32_e32 v2, v23434; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start3435; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=13436; GFX12-FAKE16-NEXT: s_wait_dscnt 0x03437; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v33438; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3439; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v0, v43440; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v33441; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3442; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, 4.0, v33443; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 13444; GFX12-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v33445; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v33446; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)3447; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff3448; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)3449; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo3450; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3451; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v33452; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v0, v33453; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3454; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v2, v33455; GFX12-FAKE16-NEXT: s_wait_storecnt 0x03456; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v43457; GFX12-FAKE16-NEXT: s_wait_dscnt 0x03458; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE3459; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v43460; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3461; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s03462; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3463; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s03464; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_13465; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end3466; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s03467; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v0, v33468; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x03469; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]3470;3471; GFX942-LABEL: local_atomic_fadd_ret_bf16:3472; GFX942: ; %bb.0:3473; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3474; GFX942-NEXT: v_and_b32_e32 v1, -4, v03475; GFX942-NEXT: ds_read_b32 v3, v13476; GFX942-NEXT: v_lshlrev_b32_e32 v2, 3, v03477; GFX942-NEXT: s_mov_b32 s0, 0xffff3478; GFX942-NEXT: v_and_b32_e32 v0, 24, v23479; GFX942-NEXT: v_lshlrev_b32_e64 v2, v2, s03480; GFX942-NEXT: v_not_b32_e32 v2, v23481; GFX942-NEXT: s_mov_b64 s[0:1], 03482; GFX942-NEXT: s_movk_i32 s2, 0x7fff3483; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start3484; GFX942-NEXT: ; =>This Inner Loop Header: Depth=13485; GFX942-NEXT: s_waitcnt lgkmcnt(0)3486; GFX942-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3487; GFX942-NEXT: s_nop 03488; GFX942-NEXT: v_add_f32_e32 v4, 4.0, v43489; GFX942-NEXT: v_bfe_u32 v5, v4, 16, 13490; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v43491; GFX942-NEXT: v_add3_u32 v5, v5, v4, s23492; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v43493; GFX942-NEXT: s_nop 13494; GFX942-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc3495; GFX942-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_13496; GFX942-NEXT: v_and_or_b32 v4, v3, v2, v43497; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v43498; GFX942-NEXT: s_waitcnt lgkmcnt(0)3499; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v33500; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]3501; GFX942-NEXT: v_mov_b32_e32 v3, v43502; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]3503; GFX942-NEXT: s_cbranch_execnz .LBB14_13504; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end3505; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]3506; GFX942-NEXT: v_lshrrev_b32_e32 v0, v0, v43507; GFX942-NEXT: s_setpc_b64 s[30:31]3508;3509; GFX11-TRUE16-LABEL: local_atomic_fadd_ret_bf16:3510; GFX11-TRUE16: ; %bb.0:3511; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3512; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, -4, v03513; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v03514; GFX11-TRUE16-NEXT: s_mov_b32 s0, 03515; GFX11-TRUE16-NEXT: ds_load_b32 v3, v13516; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff3517; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 24, v03518; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)3519; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v23520; GFX11-TRUE16-NEXT: .p2align 63521; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start3522; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=13523; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)3524; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v33525; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3526; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v43527; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v33528; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3529; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 4.0, v33530; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 13531; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v33532; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v33533; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)3534; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff3535; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo3536; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 03537; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3538; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h3539; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v53540; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3541; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v33542; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x03543; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v43544; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)3545; GFX11-TRUE16-NEXT: buffer_gl0_inv3546; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v43547; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s03548; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3549; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s03550; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_13551; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end3552; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s03553; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v33554; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]3555;3556; GFX11-FAKE16-LABEL: local_atomic_fadd_ret_bf16:3557; GFX11-FAKE16: ; %bb.0:3558; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3559; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, -4, v03560; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 3, v03561; GFX11-FAKE16-NEXT: s_mov_b32 s0, 03562; GFX11-FAKE16-NEXT: ds_load_b32 v3, v13563; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff3564; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 24, v03565; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)3566; GFX11-FAKE16-NEXT: v_not_b32_e32 v2, v23567; GFX11-FAKE16-NEXT: .p2align 63568; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start3569; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=13570; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)3571; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v33572; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3573; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v0, v43574; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v33575; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3576; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, 4.0, v33577; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 13578; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v33579; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v33580; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)3581; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff3582; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo3583; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3584; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v33585; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v0, v33586; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3587; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v2, v33588; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x03589; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v43590; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)3591; GFX11-FAKE16-NEXT: buffer_gl0_inv3592; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v43593; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s03594; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)3595; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s03596; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_13597; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end3598; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s03599; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v0, v33600; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]3601;3602; GFX10-LABEL: local_atomic_fadd_ret_bf16:3603; GFX10: ; %bb.0:3604; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3605; GFX10-NEXT: v_and_b32_e32 v1, -4, v03606; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v03607; GFX10-NEXT: s_mov_b32 s4, 03608; GFX10-NEXT: ds_read_b32 v3, v13609; GFX10-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff3610; GFX10-NEXT: v_and_b32_e32 v0, 24, v03611; GFX10-NEXT: v_not_b32_e32 v2, v23612; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start3613; GFX10-NEXT: ; =>This Inner Loop Header: Depth=13614; GFX10-NEXT: s_waitcnt lgkmcnt(0)3615; GFX10-NEXT: v_mov_b32_e32 v4, v33616; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3617; GFX10-NEXT: v_add_f32_e32 v3, 4.0, v33618; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 13619; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v33620; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v33621; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff3622; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo3623; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_13624; GFX10-NEXT: v_and_or_b32 v3, v4, v2, v33625; GFX10-NEXT: s_waitcnt_vscnt null, 0x03626; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v33627; GFX10-NEXT: s_waitcnt lgkmcnt(0)3628; GFX10-NEXT: buffer_gl0_inv3629; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v43630; GFX10-NEXT: s_or_b32 s4, vcc_lo, s43631; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s43632; GFX10-NEXT: s_cbranch_execnz .LBB14_13633; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end3634; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s43635; GFX10-NEXT: v_lshrrev_b32_e32 v0, v0, v33636; GFX10-NEXT: s_setpc_b64 s[30:31]3637;3638; GFX90A-LABEL: local_atomic_fadd_ret_bf16:3639; GFX90A: ; %bb.0:3640; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3641; GFX90A-NEXT: v_and_b32_e32 v1, -4, v03642; GFX90A-NEXT: ds_read_b32 v3, v13643; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 3, v03644; GFX90A-NEXT: s_mov_b32 s4, 0xffff3645; GFX90A-NEXT: v_and_b32_e32 v0, 24, v23646; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v2, s43647; GFX90A-NEXT: v_not_b32_e32 v2, v23648; GFX90A-NEXT: s_mov_b64 s[4:5], 03649; GFX90A-NEXT: s_movk_i32 s6, 0x7fff3650; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start3651; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=13652; GFX90A-NEXT: s_waitcnt lgkmcnt(0)3653; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3654; GFX90A-NEXT: v_add_f32_e32 v4, 4.0, v43655; GFX90A-NEXT: v_bfe_u32 v5, v4, 16, 13656; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v43657; GFX90A-NEXT: v_add3_u32 v5, v5, v4, s63658; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v43659; GFX90A-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc3660; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_13661; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v43662; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v43663; GFX90A-NEXT: s_waitcnt lgkmcnt(0)3664; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v33665; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3666; GFX90A-NEXT: v_mov_b32_e32 v3, v43667; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]3668; GFX90A-NEXT: s_cbranch_execnz .LBB14_13669; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end3670; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]3671; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v0, v43672; GFX90A-NEXT: s_setpc_b64 s[30:31]3673;3674; GFX908-LABEL: local_atomic_fadd_ret_bf16:3675; GFX908: ; %bb.0:3676; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3677; GFX908-NEXT: v_and_b32_e32 v1, -4, v03678; GFX908-NEXT: ds_read_b32 v3, v13679; GFX908-NEXT: v_lshlrev_b32_e32 v2, 3, v03680; GFX908-NEXT: s_mov_b32 s4, 0xffff3681; GFX908-NEXT: v_and_b32_e32 v0, 24, v23682; GFX908-NEXT: v_lshlrev_b32_e64 v2, v2, s43683; GFX908-NEXT: v_not_b32_e32 v2, v23684; GFX908-NEXT: s_mov_b64 s[4:5], 03685; GFX908-NEXT: s_movk_i32 s6, 0x7fff3686; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start3687; GFX908-NEXT: ; =>This Inner Loop Header: Depth=13688; GFX908-NEXT: s_waitcnt lgkmcnt(0)3689; GFX908-NEXT: v_mov_b32_e32 v4, v33690; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3691; GFX908-NEXT: v_add_f32_e32 v3, 4.0, v33692; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 13693; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v33694; GFX908-NEXT: v_add3_u32 v5, v5, v3, s63695; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v33696; GFX908-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc3697; GFX908-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_13698; GFX908-NEXT: v_and_or_b32 v3, v4, v2, v33699; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v33700; GFX908-NEXT: s_waitcnt lgkmcnt(0)3701; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v43702; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3703; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]3704; GFX908-NEXT: s_cbranch_execnz .LBB14_13705; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end3706; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]3707; GFX908-NEXT: v_lshrrev_b32_e32 v0, v0, v33708; GFX908-NEXT: s_setpc_b64 s[30:31]3709;3710; GFX8-LABEL: local_atomic_fadd_ret_bf16:3711; GFX8: ; %bb.0:3712; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3713; GFX8-NEXT: v_and_b32_e32 v1, -4, v03714; GFX8-NEXT: s_mov_b32 m0, -13715; GFX8-NEXT: ds_read_b32 v3, v13716; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v03717; GFX8-NEXT: s_mov_b32 s4, 0xffff3718; GFX8-NEXT: v_and_b32_e32 v0, 24, v23719; GFX8-NEXT: v_lshlrev_b32_e64 v2, v2, s43720; GFX8-NEXT: v_not_b32_e32 v2, v23721; GFX8-NEXT: s_mov_b64 s[4:5], 03722; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start3723; GFX8-NEXT: ; =>This Inner Loop Header: Depth=13724; GFX8-NEXT: s_waitcnt lgkmcnt(0)3725; GFX8-NEXT: v_mov_b32_e32 v4, v33726; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3727; GFX8-NEXT: v_add_f32_e32 v3, 4.0, v33728; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 13729; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v33730; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v63731; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v33732; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v33733; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc3734; GFX8-NEXT: v_and_b32_e32 v5, v4, v23735; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_13736; GFX8-NEXT: v_or_b32_e32 v3, v5, v33737; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v33738; GFX8-NEXT: s_waitcnt lgkmcnt(0)3739; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v43740; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3741; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]3742; GFX8-NEXT: s_cbranch_execnz .LBB14_13743; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end3744; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]3745; GFX8-NEXT: v_lshrrev_b32_e32 v0, v0, v33746; GFX8-NEXT: s_setpc_b64 s[30:31]3747;3748; GFX7-LABEL: local_atomic_fadd_ret_bf16:3749; GFX7: ; %bb.0:3750; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3751; GFX7-NEXT: v_and_b32_e32 v1, -4, v03752; GFX7-NEXT: s_mov_b32 m0, -13753; GFX7-NEXT: ds_read_b32 v3, v13754; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v03755; GFX7-NEXT: v_and_b32_e32 v0, 24, v23756; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v23757; GFX7-NEXT: v_not_b32_e32 v2, v23758; GFX7-NEXT: s_mov_b64 s[4:5], 03759; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start3760; GFX7-NEXT: ; =>This Inner Loop Header: Depth=13761; GFX7-NEXT: s_waitcnt lgkmcnt(0)3762; GFX7-NEXT: v_mov_b32_e32 v4, v33763; GFX7-NEXT: v_lshrrev_b32_e32 v3, v0, v43764; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v33765; GFX7-NEXT: v_add_f32_e32 v3, 4.0, v33766; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v33767; GFX7-NEXT: v_and_b32_e32 v5, v4, v23768; GFX7-NEXT: v_lshlrev_b32_e32 v3, v0, v33769; GFX7-NEXT: v_or_b32_e32 v3, v5, v33770; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v33771; GFX7-NEXT: s_waitcnt lgkmcnt(0)3772; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v43773; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3774; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]3775; GFX7-NEXT: s_cbranch_execnz .LBB14_13776; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end3777; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]3778; GFX7-NEXT: v_lshrrev_b32_e32 v0, v0, v33779; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v03780; GFX7-NEXT: s_setpc_b64 s[30:31]3781;3782; GFX6-LABEL: local_atomic_fadd_ret_bf16:3783; GFX6: ; %bb.0:3784; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3785; GFX6-NEXT: v_and_b32_e32 v1, -4, v03786; GFX6-NEXT: s_mov_b32 m0, -13787; GFX6-NEXT: ds_read_b32 v3, v13788; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v03789; GFX6-NEXT: v_and_b32_e32 v0, 24, v23790; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v23791; GFX6-NEXT: v_not_b32_e32 v2, v23792; GFX6-NEXT: s_mov_b64 s[4:5], 03793; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start3794; GFX6-NEXT: ; =>This Inner Loop Header: Depth=13795; GFX6-NEXT: s_waitcnt lgkmcnt(0)3796; GFX6-NEXT: v_mov_b32_e32 v4, v33797; GFX6-NEXT: v_lshrrev_b32_e32 v3, v0, v43798; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v33799; GFX6-NEXT: v_add_f32_e32 v3, 4.0, v33800; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v33801; GFX6-NEXT: v_and_b32_e32 v5, v4, v23802; GFX6-NEXT: v_lshlrev_b32_e32 v3, v0, v33803; GFX6-NEXT: v_or_b32_e32 v3, v5, v33804; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v33805; GFX6-NEXT: s_waitcnt lgkmcnt(0)3806; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v43807; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]3808; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]3809; GFX6-NEXT: s_cbranch_execnz .LBB14_13810; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end3811; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]3812; GFX6-NEXT: v_lshrrev_b32_e32 v0, v0, v33813; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v03814; GFX6-NEXT: s_setpc_b64 s[30:31]3815 %result = atomicrmw fadd ptr addrspace(3) %ptr, bfloat 4.0 seq_cst3816 ret bfloat %result3817}3818 3819define bfloat @local_atomic_fadd_ret_bf16__offset(ptr addrspace(3) %ptr) nounwind {3820; GFX12-TRUE16-LABEL: local_atomic_fadd_ret_bf16__offset:3821; GFX12-TRUE16: ; %bb.0:3822; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x03823; GFX12-TRUE16-NEXT: s_wait_expcnt 0x03824; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x03825; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x03826; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x03827; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v03828; GFX12-TRUE16-NEXT: s_mov_b32 s0, 03829; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)3830; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v13831; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 3, v13832; GFX12-TRUE16-NEXT: ds_load_b32 v3, v03833; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v13834; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff3835; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3836; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v23837; GFX12-TRUE16-NEXT: .LBB15_1: ; %atomicrmw.start3838; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=13839; GFX12-TRUE16-NEXT: s_wait_dscnt 0x03840; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v33841; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3842; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v43843; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v33844; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3845; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, 4.0, v33846; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 13847; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v33848; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v33849; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)3850; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff3851; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)3852; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo3853; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 03854; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3855; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h3856; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v53857; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3858; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v33859; GFX12-TRUE16-NEXT: s_wait_storecnt 0x03860; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v43861; GFX12-TRUE16-NEXT: s_wait_dscnt 0x03862; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE3863; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v43864; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3865; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s03866; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)3867; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s03868; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_13869; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end3870; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s03871; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v33872; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x03873; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]3874;3875; GFX12-FAKE16-LABEL: local_atomic_fadd_ret_bf16__offset:3876; GFX12-FAKE16: ; %bb.0:3877; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x03878; GFX12-FAKE16-NEXT: s_wait_expcnt 0x03879; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x03880; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x03881; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x03882; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v03883; GFX12-FAKE16-NEXT: s_mov_b32 s0, 03884; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)3885; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v13886; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 3, v13887; GFX12-FAKE16-NEXT: ds_load_b32 v3, v03888; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 3, v13889; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff3890; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3891; GFX12-FAKE16-NEXT: v_not_b32_e32 v2, v23892; GFX12-FAKE16-NEXT: .LBB15_1: ; %atomicrmw.start3893; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=13894; GFX12-FAKE16-NEXT: s_wait_dscnt 0x03895; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v33896; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3897; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v1, v43898; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v33899; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3900; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, 4.0, v33901; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 13902; GFX12-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v33903; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v33904; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)3905; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff3906; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)3907; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo3908; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3909; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v33910; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v1, v33911; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3912; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v2, v33913; GFX12-FAKE16-NEXT: s_wait_storecnt 0x03914; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v43915; GFX12-FAKE16-NEXT: s_wait_dscnt 0x03916; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE3917; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v43918; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3919; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s03920; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)3921; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s03922; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_13923; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end3924; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s03925; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v1, v33926; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x03927; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]3928;3929; GFX942-LABEL: local_atomic_fadd_ret_bf16__offset:3930; GFX942: ; %bb.0:3931; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3932; GFX942-NEXT: v_add_u32_e32 v0, 0xfffe, v03933; GFX942-NEXT: v_and_b32_e32 v1, -4, v03934; GFX942-NEXT: ds_read_b32 v3, v13935; GFX942-NEXT: v_and_b32_e32 v0, 3, v03936; GFX942-NEXT: v_lshlrev_b32_e32 v0, 3, v03937; GFX942-NEXT: s_mov_b32 s0, 0xffff3938; GFX942-NEXT: v_lshlrev_b32_e64 v2, v0, s03939; GFX942-NEXT: v_not_b32_e32 v2, v23940; GFX942-NEXT: s_mov_b64 s[0:1], 03941; GFX942-NEXT: s_movk_i32 s2, 0x7fff3942; GFX942-NEXT: .LBB15_1: ; %atomicrmw.start3943; GFX942-NEXT: ; =>This Inner Loop Header: Depth=13944; GFX942-NEXT: s_waitcnt lgkmcnt(0)3945; GFX942-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD3946; GFX942-NEXT: s_nop 03947; GFX942-NEXT: v_add_f32_e32 v4, 4.0, v43948; GFX942-NEXT: v_bfe_u32 v5, v4, 16, 13949; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v43950; GFX942-NEXT: v_add3_u32 v5, v5, v4, s23951; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v43952; GFX942-NEXT: s_nop 13953; GFX942-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc3954; GFX942-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_13955; GFX942-NEXT: v_and_or_b32 v4, v3, v2, v43956; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v43957; GFX942-NEXT: s_waitcnt lgkmcnt(0)3958; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v33959; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]3960; GFX942-NEXT: v_mov_b32_e32 v3, v43961; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]3962; GFX942-NEXT: s_cbranch_execnz .LBB15_13963; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end3964; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]3965; GFX942-NEXT: v_lshrrev_b32_e32 v0, v0, v43966; GFX942-NEXT: s_setpc_b64 s[30:31]3967;3968; GFX11-TRUE16-LABEL: local_atomic_fadd_ret_bf16__offset:3969; GFX11-TRUE16: ; %bb.0:3970; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)3971; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v03972; GFX11-TRUE16-NEXT: s_mov_b32 s0, 03973; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)3974; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v13975; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 3, v13976; GFX11-TRUE16-NEXT: ds_load_b32 v3, v03977; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v13978; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff3979; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)3980; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v23981; GFX11-TRUE16-NEXT: .p2align 63982; GFX11-TRUE16-NEXT: .LBB15_1: ; %atomicrmw.start3983; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=13984; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)3985; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v33986; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3987; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v43988; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v33989; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)3990; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 4.0, v33991; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 13992; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v33993; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v33994; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)3995; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff3996; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo3997; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 03998; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)3999; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h4000; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v54001; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4002; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v34003; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x04004; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v44005; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)4006; GFX11-TRUE16-NEXT: buffer_gl0_inv4007; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v44008; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s04009; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4010; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s04011; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_14012; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end4013; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s04014; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v34015; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]4016;4017; GFX11-FAKE16-LABEL: local_atomic_fadd_ret_bf16__offset:4018; GFX11-FAKE16: ; %bb.0:4019; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4020; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v04021; GFX11-FAKE16-NEXT: s_mov_b32 s0, 04022; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)4023; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v14024; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 3, v14025; GFX11-FAKE16-NEXT: ds_load_b32 v3, v04026; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 3, v14027; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff4028; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4029; GFX11-FAKE16-NEXT: v_not_b32_e32 v2, v24030; GFX11-FAKE16-NEXT: .p2align 64031; GFX11-FAKE16-NEXT: .LBB15_1: ; %atomicrmw.start4032; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=14033; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)4034; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v34035; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4036; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v1, v44037; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v34038; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4039; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, 4.0, v34040; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 14041; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v34042; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v34043; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)4044; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff4045; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo4046; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4047; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v34048; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v1, v34049; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4050; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v2, v34051; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x04052; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v44053; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)4054; GFX11-FAKE16-NEXT: buffer_gl0_inv4055; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v44056; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s04057; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4058; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s04059; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_14060; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end4061; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s04062; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v1, v34063; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]4064;4065; GFX10-LABEL: local_atomic_fadd_ret_bf16__offset:4066; GFX10: ; %bb.0:4067; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4068; GFX10-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v04069; GFX10-NEXT: s_mov_b32 s4, 04070; GFX10-NEXT: v_and_b32_e32 v0, -4, v14071; GFX10-NEXT: v_and_b32_e32 v1, 3, v14072; GFX10-NEXT: ds_read_b32 v3, v04073; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v14074; GFX10-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff4075; GFX10-NEXT: v_not_b32_e32 v2, v24076; GFX10-NEXT: .LBB15_1: ; %atomicrmw.start4077; GFX10-NEXT: ; =>This Inner Loop Header: Depth=14078; GFX10-NEXT: s_waitcnt lgkmcnt(0)4079; GFX10-NEXT: v_mov_b32_e32 v4, v34080; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4081; GFX10-NEXT: v_add_f32_e32 v3, 4.0, v34082; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 14083; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v34084; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v34085; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff4086; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo4087; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14088; GFX10-NEXT: v_and_or_b32 v3, v4, v2, v34089; GFX10-NEXT: s_waitcnt_vscnt null, 0x04090; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v34091; GFX10-NEXT: s_waitcnt lgkmcnt(0)4092; GFX10-NEXT: buffer_gl0_inv4093; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v44094; GFX10-NEXT: s_or_b32 s4, vcc_lo, s44095; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s44096; GFX10-NEXT: s_cbranch_execnz .LBB15_14097; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end4098; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s44099; GFX10-NEXT: v_lshrrev_b32_e32 v0, v1, v34100; GFX10-NEXT: s_setpc_b64 s[30:31]4101;4102; GFX90A-LABEL: local_atomic_fadd_ret_bf16__offset:4103; GFX90A: ; %bb.0:4104; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4105; GFX90A-NEXT: v_add_u32_e32 v0, 0xfffe, v04106; GFX90A-NEXT: v_and_b32_e32 v1, -4, v04107; GFX90A-NEXT: ds_read_b32 v3, v14108; GFX90A-NEXT: v_and_b32_e32 v0, 3, v04109; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 3, v04110; GFX90A-NEXT: s_mov_b32 s4, 0xffff4111; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v0, s44112; GFX90A-NEXT: v_not_b32_e32 v2, v24113; GFX90A-NEXT: s_mov_b64 s[4:5], 04114; GFX90A-NEXT: s_movk_i32 s6, 0x7fff4115; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start4116; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=14117; GFX90A-NEXT: s_waitcnt lgkmcnt(0)4118; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4119; GFX90A-NEXT: v_add_f32_e32 v4, 4.0, v44120; GFX90A-NEXT: v_bfe_u32 v5, v4, 16, 14121; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v44122; GFX90A-NEXT: v_add3_u32 v5, v5, v4, s64123; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v44124; GFX90A-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc4125; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14126; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v44127; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v44128; GFX90A-NEXT: s_waitcnt lgkmcnt(0)4129; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v34130; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4131; GFX90A-NEXT: v_mov_b32_e32 v3, v44132; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]4133; GFX90A-NEXT: s_cbranch_execnz .LBB15_14134; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end4135; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4136; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v0, v44137; GFX90A-NEXT: s_setpc_b64 s[30:31]4138;4139; GFX908-LABEL: local_atomic_fadd_ret_bf16__offset:4140; GFX908: ; %bb.0:4141; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4142; GFX908-NEXT: v_add_u32_e32 v0, 0xfffe, v04143; GFX908-NEXT: v_and_b32_e32 v1, -4, v04144; GFX908-NEXT: ds_read_b32 v3, v14145; GFX908-NEXT: v_and_b32_e32 v0, 3, v04146; GFX908-NEXT: v_lshlrev_b32_e32 v0, 3, v04147; GFX908-NEXT: s_mov_b32 s4, 0xffff4148; GFX908-NEXT: v_lshlrev_b32_e64 v2, v0, s44149; GFX908-NEXT: v_not_b32_e32 v2, v24150; GFX908-NEXT: s_mov_b64 s[4:5], 04151; GFX908-NEXT: s_movk_i32 s6, 0x7fff4152; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start4153; GFX908-NEXT: ; =>This Inner Loop Header: Depth=14154; GFX908-NEXT: s_waitcnt lgkmcnt(0)4155; GFX908-NEXT: v_mov_b32_e32 v4, v34156; GFX908-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4157; GFX908-NEXT: v_add_f32_e32 v3, 4.0, v34158; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 14159; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v34160; GFX908-NEXT: v_add3_u32 v5, v5, v3, s64161; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v34162; GFX908-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc4163; GFX908-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14164; GFX908-NEXT: v_and_or_b32 v3, v4, v2, v34165; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v34166; GFX908-NEXT: s_waitcnt lgkmcnt(0)4167; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v44168; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4169; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]4170; GFX908-NEXT: s_cbranch_execnz .LBB15_14171; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end4172; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]4173; GFX908-NEXT: v_lshrrev_b32_e32 v0, v0, v34174; GFX908-NEXT: s_setpc_b64 s[30:31]4175;4176; GFX8-LABEL: local_atomic_fadd_ret_bf16__offset:4177; GFX8: ; %bb.0:4178; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4179; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffe, v04180; GFX8-NEXT: v_and_b32_e32 v1, -4, v04181; GFX8-NEXT: s_mov_b32 m0, -14182; GFX8-NEXT: ds_read_b32 v3, v14183; GFX8-NEXT: v_and_b32_e32 v0, 3, v04184; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v04185; GFX8-NEXT: s_mov_b32 s4, 0xffff4186; GFX8-NEXT: v_lshlrev_b32_e64 v2, v0, s44187; GFX8-NEXT: v_not_b32_e32 v2, v24188; GFX8-NEXT: s_mov_b64 s[4:5], 04189; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start4190; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14191; GFX8-NEXT: s_waitcnt lgkmcnt(0)4192; GFX8-NEXT: v_mov_b32_e32 v4, v34193; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4194; GFX8-NEXT: v_add_f32_e32 v3, 4.0, v34195; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 14196; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v34197; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v64198; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v34199; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v34200; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc4201; GFX8-NEXT: v_and_b32_e32 v5, v4, v24202; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14203; GFX8-NEXT: v_or_b32_e32 v3, v5, v34204; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v1, v4, v34205; GFX8-NEXT: s_waitcnt lgkmcnt(0)4206; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v44207; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4208; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]4209; GFX8-NEXT: s_cbranch_execnz .LBB15_14210; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4211; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]4212; GFX8-NEXT: v_lshrrev_b32_e32 v0, v0, v34213; GFX8-NEXT: s_setpc_b64 s[30:31]4214;4215; GFX7-LABEL: local_atomic_fadd_ret_bf16__offset:4216; GFX7: ; %bb.0:4217; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4218; GFX7-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v04219; GFX7-NEXT: v_and_b32_e32 v0, -4, v14220; GFX7-NEXT: s_mov_b32 m0, -14221; GFX7-NEXT: ds_read_b32 v3, v04222; GFX7-NEXT: v_and_b32_e32 v1, 3, v14223; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v14224; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v14225; GFX7-NEXT: v_not_b32_e32 v2, v24226; GFX7-NEXT: s_mov_b64 s[4:5], 04227; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start4228; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14229; GFX7-NEXT: s_waitcnt lgkmcnt(0)4230; GFX7-NEXT: v_mov_b32_e32 v4, v34231; GFX7-NEXT: v_lshrrev_b32_e32 v3, v1, v44232; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v34233; GFX7-NEXT: v_add_f32_e32 v3, 4.0, v34234; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v34235; GFX7-NEXT: v_and_b32_e32 v5, v4, v24236; GFX7-NEXT: v_lshlrev_b32_e32 v3, v1, v34237; GFX7-NEXT: v_or_b32_e32 v3, v5, v34238; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v34239; GFX7-NEXT: s_waitcnt lgkmcnt(0)4240; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v44241; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4242; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]4243; GFX7-NEXT: s_cbranch_execnz .LBB15_14244; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4245; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]4246; GFX7-NEXT: v_lshrrev_b32_e32 v0, v1, v34247; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v04248; GFX7-NEXT: s_setpc_b64 s[30:31]4249;4250; GFX6-LABEL: local_atomic_fadd_ret_bf16__offset:4251; GFX6: ; %bb.0:4252; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4253; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v04254; GFX6-NEXT: v_and_b32_e32 v0, -4, v14255; GFX6-NEXT: s_mov_b32 m0, -14256; GFX6-NEXT: ds_read_b32 v3, v04257; GFX6-NEXT: v_and_b32_e32 v1, 3, v14258; GFX6-NEXT: v_lshlrev_b32_e32 v1, 3, v14259; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v14260; GFX6-NEXT: v_not_b32_e32 v2, v24261; GFX6-NEXT: s_mov_b64 s[4:5], 04262; GFX6-NEXT: .LBB15_1: ; %atomicrmw.start4263; GFX6-NEXT: ; =>This Inner Loop Header: Depth=14264; GFX6-NEXT: s_waitcnt lgkmcnt(0)4265; GFX6-NEXT: v_mov_b32_e32 v4, v34266; GFX6-NEXT: v_lshrrev_b32_e32 v3, v1, v44267; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v34268; GFX6-NEXT: v_add_f32_e32 v3, 4.0, v34269; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v34270; GFX6-NEXT: v_and_b32_e32 v5, v4, v24271; GFX6-NEXT: v_lshlrev_b32_e32 v3, v1, v34272; GFX6-NEXT: v_or_b32_e32 v3, v5, v34273; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v34274; GFX6-NEXT: s_waitcnt lgkmcnt(0)4275; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v44276; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4277; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]4278; GFX6-NEXT: s_cbranch_execnz .LBB15_14279; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end4280; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]4281; GFX6-NEXT: v_lshrrev_b32_e32 v0, v1, v34282; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v04283; GFX6-NEXT: s_setpc_b64 s[30:31]4284 %gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 327674285 %result = atomicrmw fadd ptr addrspace(3) %gep, bfloat 4.0 seq_cst4286 ret bfloat %result4287}4288 4289define void @local_atomic_fadd_noret_bf16(ptr addrspace(3) %ptr) nounwind {4290; GFX12-TRUE16-LABEL: local_atomic_fadd_noret_bf16:4291; GFX12-TRUE16: ; %bb.0:4292; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x04293; GFX12-TRUE16-NEXT: s_wait_expcnt 0x04294; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x04295; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x04296; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x04297; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, -4, v04298; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v04299; GFX12-TRUE16-NEXT: s_mov_b32 s0, 04300; GFX12-TRUE16-NEXT: ds_load_b32 v2, v14301; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff4302; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 24, v04303; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)4304; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v34305; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start4306; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=14307; GFX12-TRUE16-NEXT: s_wait_dscnt 0x04308; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)4309; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v24310; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v44311; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4312; GFX12-TRUE16-NEXT: v_add_f32_e32 v4, 4.0, v44313; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 14314; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v44315; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v44316; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)4317; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff4318; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)4319; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo4320; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 04321; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)4322; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h4323; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v54324; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4325; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v44326; GFX12-TRUE16-NEXT: s_wait_storecnt 0x04327; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v24328; GFX12-TRUE16-NEXT: s_wait_dscnt 0x04329; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE4330; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v24331; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v44332; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)4333; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s04334; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)4335; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s04336; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_14337; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end4338; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s04339; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x04340; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]4341;4342; GFX12-FAKE16-LABEL: local_atomic_fadd_noret_bf16:4343; GFX12-FAKE16: ; %bb.0:4344; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x04345; GFX12-FAKE16-NEXT: s_wait_expcnt 0x04346; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x04347; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x04348; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x04349; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, -4, v04350; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 3, v04351; GFX12-FAKE16-NEXT: s_mov_b32 s0, 04352; GFX12-FAKE16-NEXT: ds_load_b32 v2, v14353; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff4354; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 24, v04355; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)4356; GFX12-FAKE16-NEXT: v_not_b32_e32 v3, v34357; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start4358; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=14359; GFX12-FAKE16-NEXT: s_wait_dscnt 0x04360; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)4361; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v0, v24362; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v44363; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4364; GFX12-FAKE16-NEXT: v_add_f32_e32 v4, 4.0, v44365; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 14366; GFX12-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v44367; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v44368; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)4369; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff4370; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)4371; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo4372; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4373; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v44374; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v0, v44375; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4376; GFX12-FAKE16-NEXT: v_and_or_b32 v4, v2, v3, v44377; GFX12-FAKE16-NEXT: s_wait_storecnt 0x04378; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v24379; GFX12-FAKE16-NEXT: s_wait_dscnt 0x04380; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE4381; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v24382; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v44383; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)4384; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s04385; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)4386; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s04387; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB16_14388; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end4389; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s04390; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x04391; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]4392;4393; GFX942-LABEL: local_atomic_fadd_noret_bf16:4394; GFX942: ; %bb.0:4395; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4396; GFX942-NEXT: v_and_b32_e32 v1, -4, v04397; GFX942-NEXT: ds_read_b32 v3, v14398; GFX942-NEXT: v_lshlrev_b32_e32 v2, 3, v04399; GFX942-NEXT: s_mov_b32 s0, 0xffff4400; GFX942-NEXT: v_and_b32_e32 v0, 24, v24401; GFX942-NEXT: v_lshlrev_b32_e64 v2, v2, s04402; GFX942-NEXT: v_not_b32_e32 v2, v24403; GFX942-NEXT: s_mov_b64 s[0:1], 04404; GFX942-NEXT: s_movk_i32 s2, 0x7fff4405; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start4406; GFX942-NEXT: ; =>This Inner Loop Header: Depth=14407; GFX942-NEXT: s_waitcnt lgkmcnt(0)4408; GFX942-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4409; GFX942-NEXT: s_nop 04410; GFX942-NEXT: v_add_f32_e32 v4, 4.0, v44411; GFX942-NEXT: v_bfe_u32 v5, v4, 16, 14412; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v44413; GFX942-NEXT: v_add3_u32 v5, v5, v4, s24414; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v44415; GFX942-NEXT: s_nop 14416; GFX942-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc4417; GFX942-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14418; GFX942-NEXT: v_and_or_b32 v4, v3, v2, v44419; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v44420; GFX942-NEXT: s_waitcnt lgkmcnt(0)4421; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v34422; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]4423; GFX942-NEXT: v_mov_b32_e32 v3, v44424; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]4425; GFX942-NEXT: s_cbranch_execnz .LBB16_14426; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end4427; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]4428; GFX942-NEXT: s_setpc_b64 s[30:31]4429;4430; GFX11-TRUE16-LABEL: local_atomic_fadd_noret_bf16:4431; GFX11-TRUE16: ; %bb.0:4432; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4433; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, -4, v04434; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v04435; GFX11-TRUE16-NEXT: s_mov_b32 s0, 04436; GFX11-TRUE16-NEXT: ds_load_b32 v2, v14437; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff4438; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 24, v04439; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)4440; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v34441; GFX11-TRUE16-NEXT: .p2align 64442; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start4443; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=14444; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)4445; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)4446; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v24447; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v44448; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4449; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 4.0, v44450; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 14451; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v44452; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v44453; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)4454; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff4455; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo4456; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 04457; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)4458; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h4459; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v54460; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4461; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v44462; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x04463; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v24464; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)4465; GFX11-TRUE16-NEXT: buffer_gl0_inv4466; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v24467; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v44468; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s04469; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4470; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s04471; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_14472; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end4473; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s04474; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]4475;4476; GFX11-FAKE16-LABEL: local_atomic_fadd_noret_bf16:4477; GFX11-FAKE16: ; %bb.0:4478; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4479; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, -4, v04480; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 3, v04481; GFX11-FAKE16-NEXT: s_mov_b32 s0, 04482; GFX11-FAKE16-NEXT: ds_load_b32 v2, v14483; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff4484; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 24, v04485; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)4486; GFX11-FAKE16-NEXT: v_not_b32_e32 v3, v34487; GFX11-FAKE16-NEXT: .p2align 64488; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start4489; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=14490; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)4491; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)4492; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v0, v24493; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v44494; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4495; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, 4.0, v44496; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 14497; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v44498; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v44499; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)4500; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff4501; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo4502; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4503; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v44504; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v0, v44505; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4506; GFX11-FAKE16-NEXT: v_and_or_b32 v4, v2, v3, v44507; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x04508; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v24509; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)4510; GFX11-FAKE16-NEXT: buffer_gl0_inv4511; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v24512; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v44513; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s04514; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4515; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s04516; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB16_14517; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end4518; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s04519; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]4520;4521; GFX10-LABEL: local_atomic_fadd_noret_bf16:4522; GFX10: ; %bb.0:4523; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4524; GFX10-NEXT: v_and_b32_e32 v1, -4, v04525; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v04526; GFX10-NEXT: s_mov_b32 s4, 04527; GFX10-NEXT: ds_read_b32 v2, v14528; GFX10-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff4529; GFX10-NEXT: v_and_b32_e32 v0, 24, v04530; GFX10-NEXT: v_not_b32_e32 v3, v34531; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start4532; GFX10-NEXT: ; =>This Inner Loop Header: Depth=14533; GFX10-NEXT: s_waitcnt lgkmcnt(0)4534; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4535; GFX10-NEXT: v_add_f32_e32 v4, 4.0, v44536; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 14537; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v44538; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v44539; GFX10-NEXT: v_add3_u32 v5, v5, v4, 0x7fff4540; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo4541; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14542; GFX10-NEXT: v_and_or_b32 v4, v2, v3, v44543; GFX10-NEXT: s_waitcnt_vscnt null, 0x04544; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v1, v2, v44545; GFX10-NEXT: s_waitcnt lgkmcnt(0)4546; GFX10-NEXT: buffer_gl0_inv4547; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v24548; GFX10-NEXT: v_mov_b32_e32 v2, v44549; GFX10-NEXT: s_or_b32 s4, vcc_lo, s44550; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s44551; GFX10-NEXT: s_cbranch_execnz .LBB16_14552; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end4553; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s44554; GFX10-NEXT: s_setpc_b64 s[30:31]4555;4556; GFX90A-LABEL: local_atomic_fadd_noret_bf16:4557; GFX90A: ; %bb.0:4558; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4559; GFX90A-NEXT: v_and_b32_e32 v1, -4, v04560; GFX90A-NEXT: ds_read_b32 v3, v14561; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 3, v04562; GFX90A-NEXT: s_mov_b32 s4, 0xffff4563; GFX90A-NEXT: v_and_b32_e32 v0, 24, v24564; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v2, s44565; GFX90A-NEXT: v_not_b32_e32 v2, v24566; GFX90A-NEXT: s_mov_b64 s[4:5], 04567; GFX90A-NEXT: s_movk_i32 s6, 0x7fff4568; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start4569; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=14570; GFX90A-NEXT: s_waitcnt lgkmcnt(0)4571; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4572; GFX90A-NEXT: v_add_f32_e32 v4, 4.0, v44573; GFX90A-NEXT: v_bfe_u32 v5, v4, 16, 14574; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v44575; GFX90A-NEXT: v_add3_u32 v5, v5, v4, s64576; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v44577; GFX90A-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc4578; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14579; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v44580; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v44581; GFX90A-NEXT: s_waitcnt lgkmcnt(0)4582; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v34583; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4584; GFX90A-NEXT: v_mov_b32_e32 v3, v44585; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]4586; GFX90A-NEXT: s_cbranch_execnz .LBB16_14587; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end4588; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]4589; GFX90A-NEXT: s_setpc_b64 s[30:31]4590;4591; GFX908-LABEL: local_atomic_fadd_noret_bf16:4592; GFX908: ; %bb.0:4593; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4594; GFX908-NEXT: v_and_b32_e32 v1, -4, v04595; GFX908-NEXT: ds_read_b32 v3, v14596; GFX908-NEXT: v_lshlrev_b32_e32 v2, 3, v04597; GFX908-NEXT: s_mov_b32 s4, 0xffff4598; GFX908-NEXT: v_and_b32_e32 v0, 24, v24599; GFX908-NEXT: v_lshlrev_b32_e64 v2, v2, s44600; GFX908-NEXT: v_not_b32_e32 v2, v24601; GFX908-NEXT: s_mov_b64 s[4:5], 04602; GFX908-NEXT: s_movk_i32 s6, 0x7fff4603; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start4604; GFX908-NEXT: ; =>This Inner Loop Header: Depth=14605; GFX908-NEXT: s_waitcnt lgkmcnt(0)4606; GFX908-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4607; GFX908-NEXT: v_add_f32_e32 v4, 4.0, v44608; GFX908-NEXT: v_bfe_u32 v5, v4, 16, 14609; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v44610; GFX908-NEXT: v_add3_u32 v5, v5, v4, s64611; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v44612; GFX908-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc4613; GFX908-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14614; GFX908-NEXT: v_and_or_b32 v4, v3, v2, v44615; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v44616; GFX908-NEXT: s_waitcnt lgkmcnt(0)4617; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v34618; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4619; GFX908-NEXT: v_mov_b32_e32 v3, v44620; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]4621; GFX908-NEXT: s_cbranch_execnz .LBB16_14622; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end4623; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]4624; GFX908-NEXT: s_setpc_b64 s[30:31]4625;4626; GFX8-LABEL: local_atomic_fadd_noret_bf16:4627; GFX8: ; %bb.0:4628; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4629; GFX8-NEXT: v_and_b32_e32 v1, -4, v04630; GFX8-NEXT: s_mov_b32 m0, -14631; GFX8-NEXT: ds_read_b32 v3, v14632; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v04633; GFX8-NEXT: s_mov_b32 s4, 0xffff4634; GFX8-NEXT: v_and_b32_e32 v0, 24, v24635; GFX8-NEXT: v_lshlrev_b32_e64 v2, v2, s44636; GFX8-NEXT: v_not_b32_e32 v2, v24637; GFX8-NEXT: s_mov_b64 s[4:5], 04638; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start4639; GFX8-NEXT: ; =>This Inner Loop Header: Depth=14640; GFX8-NEXT: s_waitcnt lgkmcnt(0)4641; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4642; GFX8-NEXT: v_add_f32_e32 v4, 4.0, v44643; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 14644; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v44645; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v64646; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v44647; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v44648; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc4649; GFX8-NEXT: v_and_b32_e32 v5, v3, v24650; GFX8-NEXT: v_lshlrev_b32_sdwa v4, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14651; GFX8-NEXT: v_or_b32_e32 v4, v5, v44652; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v44653; GFX8-NEXT: s_waitcnt lgkmcnt(0)4654; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v34655; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4656; GFX8-NEXT: v_mov_b32_e32 v3, v44657; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]4658; GFX8-NEXT: s_cbranch_execnz .LBB16_14659; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end4660; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]4661; GFX8-NEXT: s_setpc_b64 s[30:31]4662;4663; GFX7-LABEL: local_atomic_fadd_noret_bf16:4664; GFX7: ; %bb.0:4665; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4666; GFX7-NEXT: v_and_b32_e32 v1, -4, v04667; GFX7-NEXT: s_mov_b32 m0, -14668; GFX7-NEXT: ds_read_b32 v3, v14669; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v04670; GFX7-NEXT: v_and_b32_e32 v0, 24, v24671; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v24672; GFX7-NEXT: v_not_b32_e32 v2, v24673; GFX7-NEXT: s_mov_b64 s[4:5], 04674; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start4675; GFX7-NEXT: ; =>This Inner Loop Header: Depth=14676; GFX7-NEXT: s_waitcnt lgkmcnt(0)4677; GFX7-NEXT: v_lshrrev_b32_e32 v4, v0, v34678; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v44679; GFX7-NEXT: v_add_f32_e32 v4, 4.0, v44680; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v44681; GFX7-NEXT: v_and_b32_e32 v5, v3, v24682; GFX7-NEXT: v_lshlrev_b32_e32 v4, v0, v44683; GFX7-NEXT: v_or_b32_e32 v4, v5, v44684; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v44685; GFX7-NEXT: s_waitcnt lgkmcnt(0)4686; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v34687; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4688; GFX7-NEXT: v_mov_b32_e32 v3, v44689; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]4690; GFX7-NEXT: s_cbranch_execnz .LBB16_14691; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end4692; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]4693; GFX7-NEXT: s_setpc_b64 s[30:31]4694;4695; GFX6-LABEL: local_atomic_fadd_noret_bf16:4696; GFX6: ; %bb.0:4697; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4698; GFX6-NEXT: v_and_b32_e32 v1, -4, v04699; GFX6-NEXT: s_mov_b32 m0, -14700; GFX6-NEXT: ds_read_b32 v3, v14701; GFX6-NEXT: v_lshlrev_b32_e32 v2, 3, v04702; GFX6-NEXT: v_and_b32_e32 v0, 24, v24703; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v24704; GFX6-NEXT: v_not_b32_e32 v2, v24705; GFX6-NEXT: s_mov_b64 s[4:5], 04706; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start4707; GFX6-NEXT: ; =>This Inner Loop Header: Depth=14708; GFX6-NEXT: s_waitcnt lgkmcnt(0)4709; GFX6-NEXT: v_lshrrev_b32_e32 v4, v0, v34710; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v44711; GFX6-NEXT: v_add_f32_e32 v4, 4.0, v44712; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v44713; GFX6-NEXT: v_and_b32_e32 v5, v3, v24714; GFX6-NEXT: v_lshlrev_b32_e32 v4, v0, v44715; GFX6-NEXT: v_or_b32_e32 v4, v5, v44716; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v44717; GFX6-NEXT: s_waitcnt lgkmcnt(0)4718; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v34719; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]4720; GFX6-NEXT: v_mov_b32_e32 v3, v44721; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]4722; GFX6-NEXT: s_cbranch_execnz .LBB16_14723; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end4724; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]4725; GFX6-NEXT: s_setpc_b64 s[30:31]4726 %result = atomicrmw fadd ptr addrspace(3) %ptr, bfloat 4.0 seq_cst4727 ret void4728}4729 4730define void @local_atomic_fadd_noret_bf16__offset(ptr addrspace(3) %ptr) nounwind {4731; GFX12-TRUE16-LABEL: local_atomic_fadd_noret_bf16__offset:4732; GFX12-TRUE16: ; %bb.0:4733; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x04734; GFX12-TRUE16-NEXT: s_wait_expcnt 0x04735; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x04736; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x04737; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x04738; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v04739; GFX12-TRUE16-NEXT: s_mov_b32 s0, 04740; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)4741; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v14742; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 3, v14743; GFX12-TRUE16-NEXT: ds_load_b32 v3, v04744; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v14745; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff4746; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4747; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v24748; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start4749; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=14750; GFX12-TRUE16-NEXT: s_wait_dscnt 0x04751; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v34752; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4753; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v44754; GFX12-TRUE16-NEXT: v_add_f32_e32 v4, 4.0, v44755; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)4756; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 14757; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v44758; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v44759; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff4760; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)4761; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)4762; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo4763; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 04764; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h4765; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4766; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v54767; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v44768; GFX12-TRUE16-NEXT: s_wait_storecnt 0x04769; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v34770; GFX12-TRUE16-NEXT: s_wait_dscnt 0x04771; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE4772; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v34773; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v44774; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)4775; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s04776; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)4777; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s04778; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB17_14779; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end4780; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s04781; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x04782; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]4783;4784; GFX12-FAKE16-LABEL: local_atomic_fadd_noret_bf16__offset:4785; GFX12-FAKE16: ; %bb.0:4786; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x04787; GFX12-FAKE16-NEXT: s_wait_expcnt 0x04788; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x04789; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x04790; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x04791; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v04792; GFX12-FAKE16-NEXT: s_mov_b32 s0, 04793; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)4794; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v14795; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 3, v14796; GFX12-FAKE16-NEXT: ds_load_b32 v3, v04797; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 3, v14798; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff4799; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4800; GFX12-FAKE16-NEXT: v_not_b32_e32 v2, v24801; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start4802; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=14803; GFX12-FAKE16-NEXT: s_wait_dscnt 0x04804; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v1, v34805; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4806; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v44807; GFX12-FAKE16-NEXT: v_add_f32_e32 v4, 4.0, v44808; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)4809; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 14810; GFX12-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v44811; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v44812; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff4813; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)4814; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4815; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo4816; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v44817; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4818; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v1, v44819; GFX12-FAKE16-NEXT: v_and_or_b32 v4, v3, v2, v44820; GFX12-FAKE16-NEXT: s_wait_storecnt 0x04821; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v34822; GFX12-FAKE16-NEXT: s_wait_dscnt 0x04823; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE4824; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v34825; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v44826; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)4827; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s04828; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)4829; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s04830; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB17_14831; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end4832; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s04833; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x04834; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]4835;4836; GFX942-LABEL: local_atomic_fadd_noret_bf16__offset:4837; GFX942: ; %bb.0:4838; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4839; GFX942-NEXT: v_add_u32_e32 v1, 0xfffe, v04840; GFX942-NEXT: v_and_b32_e32 v0, -4, v14841; GFX942-NEXT: ds_read_b32 v3, v04842; GFX942-NEXT: v_and_b32_e32 v1, 3, v14843; GFX942-NEXT: v_lshlrev_b32_e32 v1, 3, v14844; GFX942-NEXT: s_mov_b32 s0, 0xffff4845; GFX942-NEXT: v_lshlrev_b32_e64 v2, v1, s04846; GFX942-NEXT: v_not_b32_e32 v2, v24847; GFX942-NEXT: s_mov_b64 s[0:1], 04848; GFX942-NEXT: s_movk_i32 s2, 0x7fff4849; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start4850; GFX942-NEXT: ; =>This Inner Loop Header: Depth=14851; GFX942-NEXT: s_waitcnt lgkmcnt(0)4852; GFX942-NEXT: v_lshrrev_b32_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4853; GFX942-NEXT: s_nop 04854; GFX942-NEXT: v_add_f32_e32 v4, 4.0, v44855; GFX942-NEXT: v_bfe_u32 v5, v4, 16, 14856; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v44857; GFX942-NEXT: v_add3_u32 v5, v5, v4, s24858; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v44859; GFX942-NEXT: s_nop 14860; GFX942-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc4861; GFX942-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14862; GFX942-NEXT: v_and_or_b32 v4, v3, v2, v44863; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v44864; GFX942-NEXT: s_waitcnt lgkmcnt(0)4865; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v34866; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]4867; GFX942-NEXT: v_mov_b32_e32 v3, v44868; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]4869; GFX942-NEXT: s_cbranch_execnz .LBB17_14870; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end4871; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]4872; GFX942-NEXT: s_setpc_b64 s[30:31]4873;4874; GFX11-TRUE16-LABEL: local_atomic_fadd_noret_bf16__offset:4875; GFX11-TRUE16: ; %bb.0:4876; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4877; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v04878; GFX11-TRUE16-NEXT: s_mov_b32 s0, 04879; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)4880; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v14881; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 3, v14882; GFX11-TRUE16-NEXT: ds_load_b32 v3, v04883; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v14884; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff4885; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4886; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v24887; GFX11-TRUE16-NEXT: .p2align 64888; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start4889; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=14890; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)4891; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v34892; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4893; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v44894; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 4.0, v44895; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)4896; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 14897; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v44898; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v44899; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff4900; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)4901; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo4902; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 04903; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h4904; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4905; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v54906; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v44907; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x04908; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v34909; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)4910; GFX11-TRUE16-NEXT: buffer_gl0_inv4911; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v34912; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v44913; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s04914; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4915; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s04916; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB17_14917; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end4918; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s04919; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]4920;4921; GFX11-FAKE16-LABEL: local_atomic_fadd_noret_bf16__offset:4922; GFX11-FAKE16: ; %bb.0:4923; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4924; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v04925; GFX11-FAKE16-NEXT: s_mov_b32 s0, 04926; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)4927; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v14928; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 3, v14929; GFX11-FAKE16-NEXT: ds_load_b32 v3, v04930; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 3, v14931; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff4932; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)4933; GFX11-FAKE16-NEXT: v_not_b32_e32 v2, v24934; GFX11-FAKE16-NEXT: .p2align 64935; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start4936; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=14937; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)4938; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v1, v34939; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4940; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v44941; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, 4.0, v44942; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)4943; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 14944; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v44945; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v44946; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff4947; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4948; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo4949; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v44950; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)4951; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v1, v44952; GFX11-FAKE16-NEXT: v_and_or_b32 v4, v3, v2, v44953; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x04954; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v34955; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)4956; GFX11-FAKE16-NEXT: buffer_gl0_inv4957; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v34958; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v44959; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s04960; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)4961; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s04962; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB17_14963; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end4964; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s04965; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]4966;4967; GFX10-LABEL: local_atomic_fadd_noret_bf16__offset:4968; GFX10: ; %bb.0:4969; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)4970; GFX10-NEXT: v_add_nc_u32_e32 v1, 0xfffe, v04971; GFX10-NEXT: s_mov_b32 s4, 04972; GFX10-NEXT: v_and_b32_e32 v0, -4, v14973; GFX10-NEXT: v_and_b32_e32 v1, 3, v14974; GFX10-NEXT: ds_read_b32 v3, v04975; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v14976; GFX10-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff4977; GFX10-NEXT: v_not_b32_e32 v2, v24978; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start4979; GFX10-NEXT: ; =>This Inner Loop Header: Depth=14980; GFX10-NEXT: s_waitcnt lgkmcnt(0)4981; GFX10-NEXT: v_lshrrev_b32_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD4982; GFX10-NEXT: v_add_f32_e32 v4, 4.0, v44983; GFX10-NEXT: v_bfe_u32 v5, v4, 16, 14984; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v44985; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v44986; GFX10-NEXT: v_add3_u32 v5, v5, v4, 0x7fff4987; GFX10-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc_lo4988; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_14989; GFX10-NEXT: v_and_or_b32 v4, v3, v2, v44990; GFX10-NEXT: s_waitcnt_vscnt null, 0x04991; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v44992; GFX10-NEXT: s_waitcnt lgkmcnt(0)4993; GFX10-NEXT: buffer_gl0_inv4994; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v34995; GFX10-NEXT: v_mov_b32_e32 v3, v44996; GFX10-NEXT: s_or_b32 s4, vcc_lo, s44997; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s44998; GFX10-NEXT: s_cbranch_execnz .LBB17_14999; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end5000; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s45001; GFX10-NEXT: s_setpc_b64 s[30:31]5002;5003; GFX90A-LABEL: local_atomic_fadd_noret_bf16__offset:5004; GFX90A: ; %bb.0:5005; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5006; GFX90A-NEXT: v_add_u32_e32 v1, 0xfffe, v05007; GFX90A-NEXT: v_and_b32_e32 v0, -4, v15008; GFX90A-NEXT: ds_read_b32 v3, v05009; GFX90A-NEXT: v_and_b32_e32 v1, 3, v15010; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 3, v15011; GFX90A-NEXT: s_mov_b32 s4, 0xffff5012; GFX90A-NEXT: v_lshlrev_b32_e64 v2, v1, s45013; GFX90A-NEXT: v_not_b32_e32 v2, v25014; GFX90A-NEXT: s_mov_b64 s[4:5], 05015; GFX90A-NEXT: s_movk_i32 s6, 0x7fff5016; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start5017; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=15018; GFX90A-NEXT: s_waitcnt lgkmcnt(0)5019; GFX90A-NEXT: v_lshrrev_b32_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5020; GFX90A-NEXT: v_add_f32_e32 v4, 4.0, v45021; GFX90A-NEXT: v_bfe_u32 v5, v4, 16, 15022; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v45023; GFX90A-NEXT: v_add3_u32 v5, v5, v4, s65024; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v45025; GFX90A-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc5026; GFX90A-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15027; GFX90A-NEXT: v_and_or_b32 v4, v3, v2, v45028; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v45029; GFX90A-NEXT: s_waitcnt lgkmcnt(0)5030; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v35031; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5032; GFX90A-NEXT: v_mov_b32_e32 v3, v45033; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]5034; GFX90A-NEXT: s_cbranch_execnz .LBB17_15035; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end5036; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5037; GFX90A-NEXT: s_setpc_b64 s[30:31]5038;5039; GFX908-LABEL: local_atomic_fadd_noret_bf16__offset:5040; GFX908: ; %bb.0:5041; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5042; GFX908-NEXT: v_add_u32_e32 v1, 0xfffe, v05043; GFX908-NEXT: v_and_b32_e32 v0, -4, v15044; GFX908-NEXT: ds_read_b32 v3, v05045; GFX908-NEXT: v_and_b32_e32 v1, 3, v15046; GFX908-NEXT: v_lshlrev_b32_e32 v1, 3, v15047; GFX908-NEXT: s_mov_b32 s4, 0xffff5048; GFX908-NEXT: v_lshlrev_b32_e64 v2, v1, s45049; GFX908-NEXT: v_not_b32_e32 v2, v25050; GFX908-NEXT: s_mov_b64 s[4:5], 05051; GFX908-NEXT: s_movk_i32 s6, 0x7fff5052; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start5053; GFX908-NEXT: ; =>This Inner Loop Header: Depth=15054; GFX908-NEXT: s_waitcnt lgkmcnt(0)5055; GFX908-NEXT: v_lshrrev_b32_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5056; GFX908-NEXT: v_add_f32_e32 v4, 4.0, v45057; GFX908-NEXT: v_bfe_u32 v5, v4, 16, 15058; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v45059; GFX908-NEXT: v_add3_u32 v5, v5, v4, s65060; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v45061; GFX908-NEXT: v_cndmask_b32_e32 v4, v5, v6, vcc5062; GFX908-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15063; GFX908-NEXT: v_and_or_b32 v4, v3, v2, v45064; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v45065; GFX908-NEXT: s_waitcnt lgkmcnt(0)5066; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v35067; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5068; GFX908-NEXT: v_mov_b32_e32 v3, v45069; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]5070; GFX908-NEXT: s_cbranch_execnz .LBB17_15071; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end5072; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]5073; GFX908-NEXT: s_setpc_b64 s[30:31]5074;5075; GFX8-LABEL: local_atomic_fadd_noret_bf16__offset:5076; GFX8: ; %bb.0:5077; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5078; GFX8-NEXT: v_add_u32_e32 v1, vcc, 0xfffe, v05079; GFX8-NEXT: v_and_b32_e32 v0, -4, v15080; GFX8-NEXT: s_mov_b32 m0, -15081; GFX8-NEXT: ds_read_b32 v3, v05082; GFX8-NEXT: v_and_b32_e32 v1, 3, v15083; GFX8-NEXT: v_lshlrev_b32_e32 v1, 3, v15084; GFX8-NEXT: s_mov_b32 s4, 0xffff5085; GFX8-NEXT: v_lshlrev_b32_e64 v2, v1, s45086; GFX8-NEXT: v_not_b32_e32 v2, v25087; GFX8-NEXT: s_mov_b64 s[4:5], 05088; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start5089; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15090; GFX8-NEXT: s_waitcnt lgkmcnt(0)5091; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD5092; GFX8-NEXT: v_add_f32_e32 v4, 4.0, v45093; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 15094; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v45095; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v65096; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v45097; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v45098; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc5099; GFX8-NEXT: v_and_b32_e32 v5, v3, v25100; GFX8-NEXT: v_lshlrev_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15101; GFX8-NEXT: v_or_b32_e32 v4, v5, v45102; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v45103; GFX8-NEXT: s_waitcnt lgkmcnt(0)5104; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v35105; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5106; GFX8-NEXT: v_mov_b32_e32 v3, v45107; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]5108; GFX8-NEXT: s_cbranch_execnz .LBB17_15109; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5110; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]5111; GFX8-NEXT: s_setpc_b64 s[30:31]5112;5113; GFX7-LABEL: local_atomic_fadd_noret_bf16__offset:5114; GFX7: ; %bb.0:5115; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5116; GFX7-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v05117; GFX7-NEXT: v_and_b32_e32 v0, -4, v15118; GFX7-NEXT: s_mov_b32 m0, -15119; GFX7-NEXT: ds_read_b32 v3, v05120; GFX7-NEXT: v_and_b32_e32 v1, 3, v15121; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v15122; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v15123; GFX7-NEXT: v_not_b32_e32 v2, v25124; GFX7-NEXT: s_mov_b64 s[4:5], 05125; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start5126; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15127; GFX7-NEXT: s_waitcnt lgkmcnt(0)5128; GFX7-NEXT: v_lshrrev_b32_e32 v4, v1, v35129; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v45130; GFX7-NEXT: v_add_f32_e32 v4, 4.0, v45131; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v45132; GFX7-NEXT: v_and_b32_e32 v5, v3, v25133; GFX7-NEXT: v_lshlrev_b32_e32 v4, v1, v45134; GFX7-NEXT: v_or_b32_e32 v4, v5, v45135; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v45136; GFX7-NEXT: s_waitcnt lgkmcnt(0)5137; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v35138; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5139; GFX7-NEXT: v_mov_b32_e32 v3, v45140; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]5141; GFX7-NEXT: s_cbranch_execnz .LBB17_15142; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5143; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]5144; GFX7-NEXT: s_setpc_b64 s[30:31]5145;5146; GFX6-LABEL: local_atomic_fadd_noret_bf16__offset:5147; GFX6: ; %bb.0:5148; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5149; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v05150; GFX6-NEXT: v_and_b32_e32 v0, -4, v15151; GFX6-NEXT: s_mov_b32 m0, -15152; GFX6-NEXT: ds_read_b32 v3, v05153; GFX6-NEXT: v_and_b32_e32 v1, 3, v15154; GFX6-NEXT: v_lshlrev_b32_e32 v1, 3, v15155; GFX6-NEXT: v_lshl_b32_e32 v2, 0xffff, v15156; GFX6-NEXT: v_not_b32_e32 v2, v25157; GFX6-NEXT: s_mov_b64 s[4:5], 05158; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start5159; GFX6-NEXT: ; =>This Inner Loop Header: Depth=15160; GFX6-NEXT: s_waitcnt lgkmcnt(0)5161; GFX6-NEXT: v_lshrrev_b32_e32 v4, v1, v35162; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v45163; GFX6-NEXT: v_add_f32_e32 v4, 4.0, v45164; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v45165; GFX6-NEXT: v_and_b32_e32 v5, v3, v25166; GFX6-NEXT: v_lshlrev_b32_e32 v4, v1, v45167; GFX6-NEXT: v_or_b32_e32 v4, v5, v45168; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v45169; GFX6-NEXT: s_waitcnt lgkmcnt(0)5170; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v35171; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5172; GFX6-NEXT: v_mov_b32_e32 v3, v45173; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]5174; GFX6-NEXT: s_cbranch_execnz .LBB17_15175; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end5176; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]5177; GFX6-NEXT: s_setpc_b64 s[30:31]5178 %gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 327675179 %unused = atomicrmw fadd ptr addrspace(3) %gep, bfloat 4.0 seq_cst5180 ret void5181}5182 5183define bfloat @local_atomic_fadd_ret_bf16__offset__align4(ptr addrspace(3) %ptr) nounwind {5184; GFX12-TRUE16-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5185; GFX12-TRUE16: ; %bb.0:5186; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x05187; GFX12-TRUE16-NEXT: s_wait_expcnt 0x05188; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x05189; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x05190; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x05191; GFX12-TRUE16-NEXT: ds_load_b32 v1, v0 offset:655345192; GFX12-TRUE16-NEXT: s_mov_b32 s0, 05193; GFX12-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start5194; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=15195; GFX12-TRUE16-NEXT: s_wait_dscnt 0x05196; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v15197; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5198; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v25199; GFX12-TRUE16-NEXT: v_add_f32_e32 v1, 4.0, v15200; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)5201; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 15202; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v15203; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v15204; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff5205; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)5206; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)5207; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo5208; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 05209; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h5210; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5211; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v35212; GFX12-TRUE16-NEXT: s_wait_storecnt 0x05213; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:655345214; GFX12-TRUE16-NEXT: s_wait_dscnt 0x05215; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE5216; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v25217; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5218; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s05219; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5220; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s05221; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_15222; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end5223; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s05224; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l5225; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x05226; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]5227;5228; GFX12-FAKE16-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5229; GFX12-FAKE16: ; %bb.0:5230; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x05231; GFX12-FAKE16-NEXT: s_wait_expcnt 0x05232; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x05233; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x05234; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x05235; GFX12-FAKE16-NEXT: ds_load_b32 v1, v0 offset:655345236; GFX12-FAKE16-NEXT: s_mov_b32 s0, 05237; GFX12-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start5238; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=15239; GFX12-FAKE16-NEXT: s_wait_dscnt 0x05240; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v15241; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5242; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v25243; GFX12-FAKE16-NEXT: v_add_f32_e32 v1, 4.0, v15244; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)5245; GFX12-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 15246; GFX12-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v15247; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v15248; GFX12-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff5249; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)5250; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5251; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo5252; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v15253; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5254; GFX12-FAKE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v15255; GFX12-FAKE16-NEXT: s_wait_storecnt 0x05256; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:655345257; GFX12-FAKE16-NEXT: s_wait_dscnt 0x05258; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE5259; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v25260; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5261; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s05262; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5263; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s05264; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_15265; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end5266; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s05267; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v15268; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x05269; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]5270;5271; GFX942-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5272; GFX942: ; %bb.0:5273; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5274; GFX942-NEXT: ds_read_b32 v2, v0 offset:655345275; GFX942-NEXT: s_mov_b64 s[0:1], 05276; GFX942-NEXT: s_movk_i32 s2, 0x7fff5277; GFX942-NEXT: s_mov_b32 s3, 0xffff00005278; GFX942-NEXT: .LBB18_1: ; %atomicrmw.start5279; GFX942-NEXT: ; =>This Inner Loop Header: Depth=15280; GFX942-NEXT: s_waitcnt lgkmcnt(0)5281; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v25282; GFX942-NEXT: v_add_f32_e32 v1, 4.0, v15283; GFX942-NEXT: v_bfe_u32 v3, v1, 16, 15284; GFX942-NEXT: v_or_b32_e32 v4, 0x400000, v15285; GFX942-NEXT: v_add3_u32 v3, v3, v1, s25286; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v15287; GFX942-NEXT: s_nop 15288; GFX942-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc5289; GFX942-NEXT: v_lshrrev_b32_e32 v1, 16, v15290; GFX942-NEXT: v_and_or_b32 v1, v2, s3, v15291; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655345292; GFX942-NEXT: s_waitcnt lgkmcnt(0)5293; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v25294; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]5295; GFX942-NEXT: v_mov_b32_e32 v2, v15296; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]5297; GFX942-NEXT: s_cbranch_execnz .LBB18_15298; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end5299; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]5300; GFX942-NEXT: v_mov_b32_e32 v0, v15301; GFX942-NEXT: s_setpc_b64 s[30:31]5302;5303; GFX11-TRUE16-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5304; GFX11-TRUE16: ; %bb.0:5305; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5306; GFX11-TRUE16-NEXT: ds_load_b32 v1, v0 offset:655345307; GFX11-TRUE16-NEXT: s_mov_b32 s0, 05308; GFX11-TRUE16-NEXT: .p2align 65309; GFX11-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start5310; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=15311; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)5312; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v15313; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5314; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v25315; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 4.0, v15316; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)5317; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 15318; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v15319; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v15320; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff5321; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)5322; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo5323; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 05324; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h5325; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5326; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v35327; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x05328; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:655345329; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)5330; GFX11-TRUE16-NEXT: buffer_gl0_inv5331; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v25332; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s05333; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5334; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s05335; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_15336; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end5337; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s05338; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l5339; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]5340;5341; GFX11-FAKE16-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5342; GFX11-FAKE16: ; %bb.0:5343; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5344; GFX11-FAKE16-NEXT: ds_load_b32 v1, v0 offset:655345345; GFX11-FAKE16-NEXT: s_mov_b32 s0, 05346; GFX11-FAKE16-NEXT: .p2align 65347; GFX11-FAKE16-NEXT: .LBB18_1: ; %atomicrmw.start5348; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=15349; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)5350; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v15351; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5352; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v25353; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, 4.0, v15354; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)5355; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 15356; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v15357; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v15358; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff5359; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5360; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo5361; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v15362; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)5363; GFX11-FAKE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v15364; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x05365; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:655345366; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)5367; GFX11-FAKE16-NEXT: buffer_gl0_inv5368; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v25369; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s05370; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5371; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s05372; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_15373; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end5374; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s05375; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v15376; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]5377;5378; GFX10-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5379; GFX10: ; %bb.0:5380; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5381; GFX10-NEXT: ds_read_b32 v1, v0 offset:655345382; GFX10-NEXT: s_mov_b32 s4, 05383; GFX10-NEXT: .LBB18_1: ; %atomicrmw.start5384; GFX10-NEXT: ; =>This Inner Loop Header: Depth=15385; GFX10-NEXT: s_waitcnt lgkmcnt(0)5386; GFX10-NEXT: v_mov_b32_e32 v2, v15387; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v25388; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v15389; GFX10-NEXT: v_bfe_u32 v3, v1, 16, 15390; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v15391; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v15392; GFX10-NEXT: v_add3_u32 v3, v3, v1, 0x7fff5393; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo5394; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v15395; GFX10-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v15396; GFX10-NEXT: s_waitcnt_vscnt null, 0x05397; GFX10-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655345398; GFX10-NEXT: s_waitcnt lgkmcnt(0)5399; GFX10-NEXT: buffer_gl0_inv5400; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v25401; GFX10-NEXT: s_or_b32 s4, vcc_lo, s45402; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s45403; GFX10-NEXT: s_cbranch_execnz .LBB18_15404; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end5405; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s45406; GFX10-NEXT: v_mov_b32_e32 v0, v15407; GFX10-NEXT: s_setpc_b64 s[30:31]5408;5409; GFX90A-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5410; GFX90A: ; %bb.0:5411; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5412; GFX90A-NEXT: ds_read_b32 v2, v0 offset:655345413; GFX90A-NEXT: s_mov_b64 s[4:5], 05414; GFX90A-NEXT: s_movk_i32 s6, 0x7fff5415; GFX90A-NEXT: s_mov_b32 s7, 0xffff00005416; GFX90A-NEXT: .LBB18_1: ; %atomicrmw.start5417; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=15418; GFX90A-NEXT: s_waitcnt lgkmcnt(0)5419; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v25420; GFX90A-NEXT: v_add_f32_e32 v1, 4.0, v15421; GFX90A-NEXT: v_bfe_u32 v3, v1, 16, 15422; GFX90A-NEXT: v_or_b32_e32 v4, 0x400000, v15423; GFX90A-NEXT: v_add3_u32 v3, v3, v1, s65424; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v15425; GFX90A-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc5426; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 16, v15427; GFX90A-NEXT: v_and_or_b32 v1, v2, s7, v15428; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655345429; GFX90A-NEXT: s_waitcnt lgkmcnt(0)5430; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v25431; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5432; GFX90A-NEXT: v_mov_b32_e32 v2, v15433; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]5434; GFX90A-NEXT: s_cbranch_execnz .LBB18_15435; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end5436; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5437; GFX90A-NEXT: v_mov_b32_e32 v0, v15438; GFX90A-NEXT: s_setpc_b64 s[30:31]5439;5440; GFX908-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5441; GFX908: ; %bb.0:5442; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5443; GFX908-NEXT: ds_read_b32 v1, v0 offset:655345444; GFX908-NEXT: s_mov_b64 s[4:5], 05445; GFX908-NEXT: s_movk_i32 s6, 0x7fff5446; GFX908-NEXT: s_mov_b32 s7, 0xffff00005447; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start5448; GFX908-NEXT: ; =>This Inner Loop Header: Depth=15449; GFX908-NEXT: s_waitcnt lgkmcnt(0)5450; GFX908-NEXT: v_mov_b32_e32 v2, v15451; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v25452; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v15453; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 15454; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v15455; GFX908-NEXT: v_add3_u32 v3, v3, v1, s65456; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v15457; GFX908-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc5458; GFX908-NEXT: v_lshrrev_b32_e32 v1, 16, v15459; GFX908-NEXT: v_and_or_b32 v1, v2, s7, v15460; GFX908-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655345461; GFX908-NEXT: s_waitcnt lgkmcnt(0)5462; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v25463; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5464; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]5465; GFX908-NEXT: s_cbranch_execnz .LBB18_15466; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end5467; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]5468; GFX908-NEXT: v_mov_b32_e32 v0, v15469; GFX908-NEXT: s_setpc_b64 s[30:31]5470;5471; GFX8-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5472; GFX8: ; %bb.0:5473; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5474; GFX8-NEXT: s_mov_b32 m0, -15475; GFX8-NEXT: ds_read_b32 v1, v0 offset:655345476; GFX8-NEXT: s_mov_b64 s[4:5], 05477; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start5478; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15479; GFX8-NEXT: s_waitcnt lgkmcnt(0)5480; GFX8-NEXT: v_mov_b32_e32 v2, v15481; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v25482; GFX8-NEXT: v_add_f32_e32 v1, 4.0, v15483; GFX8-NEXT: v_bfe_u32 v4, v1, 16, 15484; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v15485; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v45486; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v15487; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v15488; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v25489; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc5490; GFX8-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15491; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655345492; GFX8-NEXT: s_waitcnt lgkmcnt(0)5493; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v25494; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5495; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]5496; GFX8-NEXT: s_cbranch_execnz .LBB18_15497; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5498; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]5499; GFX8-NEXT: v_mov_b32_e32 v0, v15500; GFX8-NEXT: s_setpc_b64 s[30:31]5501;5502; GFX7-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5503; GFX7: ; %bb.0:5504; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5505; GFX7-NEXT: s_mov_b32 m0, -15506; GFX7-NEXT: ds_read_b32 v1, v0 offset:655345507; GFX7-NEXT: s_mov_b64 s[4:5], 05508; GFX7-NEXT: .LBB18_1: ; %atomicrmw.start5509; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15510; GFX7-NEXT: s_waitcnt lgkmcnt(0)5511; GFX7-NEXT: v_mov_b32_e32 v2, v15512; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v25513; GFX7-NEXT: v_add_f32_e32 v1, 4.0, v15514; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v25515; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v15516; GFX7-NEXT: v_or_b32_e32 v1, v3, v15517; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:655345518; GFX7-NEXT: s_waitcnt lgkmcnt(0)5519; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v25520; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5521; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]5522; GFX7-NEXT: s_cbranch_execnz .LBB18_15523; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5524; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]5525; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v15526; GFX7-NEXT: s_setpc_b64 s[30:31]5527;5528; GFX6-LABEL: local_atomic_fadd_ret_bf16__offset__align4:5529; GFX6: ; %bb.0:5530; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5531; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v05532; GFX6-NEXT: s_mov_b32 m0, -15533; GFX6-NEXT: ds_read_b32 v1, v05534; GFX6-NEXT: s_mov_b64 s[4:5], 05535; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start5536; GFX6-NEXT: ; =>This Inner Loop Header: Depth=15537; GFX6-NEXT: s_waitcnt lgkmcnt(0)5538; GFX6-NEXT: v_mov_b32_e32 v2, v15539; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v25540; GFX6-NEXT: v_add_f32_e32 v1, 4.0, v15541; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v25542; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v15543; GFX6-NEXT: v_or_b32_e32 v1, v3, v15544; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v15545; GFX6-NEXT: s_waitcnt lgkmcnt(0)5546; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v25547; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5548; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]5549; GFX6-NEXT: s_cbranch_execnz .LBB18_15550; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end5551; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]5552; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v15553; GFX6-NEXT: s_setpc_b64 s[30:31]5554 %gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 327675555 %result = atomicrmw fadd ptr addrspace(3) %gep, bfloat 4.0 seq_cst, align 45556 ret bfloat %result5557}5558 5559define void @local_atomic_fadd_noret_bf16__offset__align4(ptr addrspace(3) %ptr) nounwind {5560; GFX12-TRUE16-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5561; GFX12-TRUE16: ; %bb.0:5562; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x05563; GFX12-TRUE16-NEXT: s_wait_expcnt 0x05564; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x05565; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x05566; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x05567; GFX12-TRUE16-NEXT: ds_load_b32 v1, v0 offset:655345568; GFX12-TRUE16-NEXT: s_mov_b32 s0, 05569; GFX12-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start5570; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=15571; GFX12-TRUE16-NEXT: s_wait_dscnt 0x05572; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v15573; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5574; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, 4.0, v25575; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 15576; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v25577; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v25578; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)5579; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff5580; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)5581; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo5582; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 05583; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)5584; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h5585; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v35586; GFX12-TRUE16-NEXT: s_wait_storecnt 0x05587; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:655345588; GFX12-TRUE16-NEXT: s_wait_dscnt 0x05589; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE5590; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v15591; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v25592; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5593; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s05594; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)5595; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s05596; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB19_15597; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end5598; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s05599; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x05600; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]5601;5602; GFX12-FAKE16-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5603; GFX12-FAKE16: ; %bb.0:5604; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x05605; GFX12-FAKE16-NEXT: s_wait_expcnt 0x05606; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x05607; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x05608; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x05609; GFX12-FAKE16-NEXT: ds_load_b32 v1, v0 offset:655345610; GFX12-FAKE16-NEXT: s_mov_b32 s0, 05611; GFX12-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start5612; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=15613; GFX12-FAKE16-NEXT: s_wait_dscnt 0x05614; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v15615; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5616; GFX12-FAKE16-NEXT: v_add_f32_e32 v2, 4.0, v25617; GFX12-FAKE16-NEXT: v_bfe_u32 v3, v2, 16, 15618; GFX12-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v25619; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v25620; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)5621; GFX12-FAKE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff5622; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)5623; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo5624; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5625; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v25626; GFX12-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v25627; GFX12-FAKE16-NEXT: s_wait_storecnt 0x05628; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:655345629; GFX12-FAKE16-NEXT: s_wait_dscnt 0x05630; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE5631; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v15632; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v25633; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5634; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s05635; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)5636; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s05637; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB19_15638; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end5639; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s05640; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x05641; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]5642;5643; GFX942-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5644; GFX942: ; %bb.0:5645; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5646; GFX942-NEXT: ds_read_b32 v1, v0 offset:655345647; GFX942-NEXT: s_mov_b64 s[0:1], 05648; GFX942-NEXT: s_movk_i32 s2, 0x7fff5649; GFX942-NEXT: s_mov_b32 s3, 0xffff00005650; GFX942-NEXT: .LBB19_1: ; %atomicrmw.start5651; GFX942-NEXT: ; =>This Inner Loop Header: Depth=15652; GFX942-NEXT: s_waitcnt lgkmcnt(0)5653; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v15654; GFX942-NEXT: v_add_f32_e32 v2, 4.0, v25655; GFX942-NEXT: v_bfe_u32 v3, v2, 16, 15656; GFX942-NEXT: v_or_b32_e32 v4, 0x400000, v25657; GFX942-NEXT: v_add3_u32 v3, v3, v2, s25658; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v25659; GFX942-NEXT: s_nop 15660; GFX942-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc5661; GFX942-NEXT: v_lshrrev_b32_e32 v2, 16, v25662; GFX942-NEXT: v_and_or_b32 v2, v1, s3, v25663; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655345664; GFX942-NEXT: s_waitcnt lgkmcnt(0)5665; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15666; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]5667; GFX942-NEXT: v_mov_b32_e32 v1, v25668; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]5669; GFX942-NEXT: s_cbranch_execnz .LBB19_15670; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end5671; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]5672; GFX942-NEXT: s_setpc_b64 s[30:31]5673;5674; GFX11-TRUE16-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5675; GFX11-TRUE16: ; %bb.0:5676; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5677; GFX11-TRUE16-NEXT: ds_load_b32 v1, v0 offset:655345678; GFX11-TRUE16-NEXT: s_mov_b32 s0, 05679; GFX11-TRUE16-NEXT: .p2align 65680; GFX11-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start5681; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=15682; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)5683; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v15684; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5685; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 4.0, v25686; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 15687; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v25688; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v25689; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)5690; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff5691; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo5692; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 05693; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)5694; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h5695; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v35696; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x05697; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:655345698; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)5699; GFX11-TRUE16-NEXT: buffer_gl0_inv5700; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v15701; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v25702; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s05703; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5704; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s05705; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB19_15706; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end5707; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s05708; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]5709;5710; GFX11-FAKE16-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5711; GFX11-FAKE16: ; %bb.0:5712; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5713; GFX11-FAKE16-NEXT: ds_load_b32 v1, v0 offset:655345714; GFX11-FAKE16-NEXT: s_mov_b32 s0, 05715; GFX11-FAKE16-NEXT: .p2align 65716; GFX11-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start5717; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=15718; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)5719; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v15720; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5721; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, 4.0, v25722; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v2, 16, 15723; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v25724; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v25725; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)5726; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff5727; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo5728; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)5729; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v25730; GFX11-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v25731; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x05732; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:655345733; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)5734; GFX11-FAKE16-NEXT: buffer_gl0_inv5735; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v15736; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v25737; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s05738; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5739; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s05740; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB19_15741; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end5742; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s05743; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]5744;5745; GFX10-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5746; GFX10: ; %bb.0:5747; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5748; GFX10-NEXT: ds_read_b32 v1, v0 offset:655345749; GFX10-NEXT: s_mov_b32 s4, 05750; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start5751; GFX10-NEXT: ; =>This Inner Loop Header: Depth=15752; GFX10-NEXT: s_waitcnt lgkmcnt(0)5753; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v15754; GFX10-NEXT: v_add_f32_e32 v2, 4.0, v25755; GFX10-NEXT: v_bfe_u32 v3, v2, 16, 15756; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v25757; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v25758; GFX10-NEXT: v_add3_u32 v3, v3, v2, 0x7fff5759; GFX10-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo5760; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v25761; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v25762; GFX10-NEXT: s_waitcnt_vscnt null, 0x05763; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655345764; GFX10-NEXT: s_waitcnt lgkmcnt(0)5765; GFX10-NEXT: buffer_gl0_inv5766; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v15767; GFX10-NEXT: v_mov_b32_e32 v1, v25768; GFX10-NEXT: s_or_b32 s4, vcc_lo, s45769; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s45770; GFX10-NEXT: s_cbranch_execnz .LBB19_15771; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end5772; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s45773; GFX10-NEXT: s_setpc_b64 s[30:31]5774;5775; GFX90A-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5776; GFX90A: ; %bb.0:5777; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5778; GFX90A-NEXT: ds_read_b32 v1, v0 offset:655345779; GFX90A-NEXT: s_mov_b64 s[4:5], 05780; GFX90A-NEXT: s_movk_i32 s6, 0x7fff5781; GFX90A-NEXT: s_mov_b32 s7, 0xffff00005782; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start5783; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=15784; GFX90A-NEXT: s_waitcnt lgkmcnt(0)5785; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v15786; GFX90A-NEXT: v_add_f32_e32 v2, 4.0, v25787; GFX90A-NEXT: v_bfe_u32 v3, v2, 16, 15788; GFX90A-NEXT: v_or_b32_e32 v4, 0x400000, v25789; GFX90A-NEXT: v_add3_u32 v3, v3, v2, s65790; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v25791; GFX90A-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc5792; GFX90A-NEXT: v_lshrrev_b32_e32 v2, 16, v25793; GFX90A-NEXT: v_and_or_b32 v2, v1, s7, v25794; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655345795; GFX90A-NEXT: s_waitcnt lgkmcnt(0)5796; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15797; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5798; GFX90A-NEXT: v_mov_b32_e32 v1, v25799; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]5800; GFX90A-NEXT: s_cbranch_execnz .LBB19_15801; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end5802; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]5803; GFX90A-NEXT: s_setpc_b64 s[30:31]5804;5805; GFX908-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5806; GFX908: ; %bb.0:5807; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5808; GFX908-NEXT: ds_read_b32 v1, v0 offset:655345809; GFX908-NEXT: s_mov_b64 s[4:5], 05810; GFX908-NEXT: s_movk_i32 s6, 0x7fff5811; GFX908-NEXT: s_mov_b32 s7, 0xffff00005812; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start5813; GFX908-NEXT: ; =>This Inner Loop Header: Depth=15814; GFX908-NEXT: s_waitcnt lgkmcnt(0)5815; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v15816; GFX908-NEXT: v_add_f32_e32 v2, 4.0, v25817; GFX908-NEXT: v_bfe_u32 v3, v2, 16, 15818; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v25819; GFX908-NEXT: v_add3_u32 v3, v3, v2, s65820; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v25821; GFX908-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc5822; GFX908-NEXT: v_lshrrev_b32_e32 v2, 16, v25823; GFX908-NEXT: v_and_or_b32 v2, v1, s7, v25824; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655345825; GFX908-NEXT: s_waitcnt lgkmcnt(0)5826; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15827; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5828; GFX908-NEXT: v_mov_b32_e32 v1, v25829; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]5830; GFX908-NEXT: s_cbranch_execnz .LBB19_15831; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end5832; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]5833; GFX908-NEXT: s_setpc_b64 s[30:31]5834;5835; GFX8-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5836; GFX8: ; %bb.0:5837; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5838; GFX8-NEXT: s_mov_b32 m0, -15839; GFX8-NEXT: ds_read_b32 v1, v0 offset:655345840; GFX8-NEXT: s_mov_b64 s[4:5], 05841; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start5842; GFX8-NEXT: ; =>This Inner Loop Header: Depth=15843; GFX8-NEXT: s_waitcnt lgkmcnt(0)5844; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v15845; GFX8-NEXT: v_add_f32_e32 v2, 4.0, v25846; GFX8-NEXT: v_bfe_u32 v4, v2, 16, 15847; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v25848; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v45849; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v25850; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v25851; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v15852; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc5853; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_15854; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655345855; GFX8-NEXT: s_waitcnt lgkmcnt(0)5856; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15857; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5858; GFX8-NEXT: v_mov_b32_e32 v1, v25859; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]5860; GFX8-NEXT: s_cbranch_execnz .LBB19_15861; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end5862; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]5863; GFX8-NEXT: s_setpc_b64 s[30:31]5864;5865; GFX7-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5866; GFX7: ; %bb.0:5867; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5868; GFX7-NEXT: s_mov_b32 m0, -15869; GFX7-NEXT: ds_read_b32 v1, v0 offset:655345870; GFX7-NEXT: s_mov_b64 s[4:5], 05871; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start5872; GFX7-NEXT: ; =>This Inner Loop Header: Depth=15873; GFX7-NEXT: s_waitcnt lgkmcnt(0)5874; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v15875; GFX7-NEXT: v_add_f32_e32 v2, 4.0, v25876; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v15877; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v25878; GFX7-NEXT: v_or_b32_e32 v2, v3, v25879; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2 offset:655345880; GFX7-NEXT: s_waitcnt lgkmcnt(0)5881; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15882; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5883; GFX7-NEXT: v_mov_b32_e32 v1, v25884; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]5885; GFX7-NEXT: s_cbranch_execnz .LBB19_15886; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end5887; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]5888; GFX7-NEXT: s_setpc_b64 s[30:31]5889;5890; GFX6-LABEL: local_atomic_fadd_noret_bf16__offset__align4:5891; GFX6: ; %bb.0:5892; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5893; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v05894; GFX6-NEXT: s_mov_b32 m0, -15895; GFX6-NEXT: ds_read_b32 v1, v05896; GFX6-NEXT: s_mov_b64 s[4:5], 05897; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start5898; GFX6-NEXT: ; =>This Inner Loop Header: Depth=15899; GFX6-NEXT: s_waitcnt lgkmcnt(0)5900; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v15901; GFX6-NEXT: v_add_f32_e32 v2, 4.0, v25902; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v15903; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v25904; GFX6-NEXT: v_or_b32_e32 v2, v3, v25905; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v25906; GFX6-NEXT: s_waitcnt lgkmcnt(0)5907; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v15908; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]5909; GFX6-NEXT: v_mov_b32_e32 v1, v25910; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]5911; GFX6-NEXT: s_cbranch_execnz .LBB19_15912; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end5913; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]5914; GFX6-NEXT: s_setpc_b64 s[30:31]5915 %gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 327675916 %unused = atomicrmw fadd ptr addrspace(3) %gep, bfloat 4.0 seq_cst, align 45917 ret void5918}5919 5920; --------------------------------------------------------------------5921; <2 x half>5922; --------------------------------------------------------------------5923 5924define <2 x half> @local_atomic_fadd_ret_v2f16(ptr addrspace(3) %ptr, <2 x half> %val) {5925; GFX12-LABEL: local_atomic_fadd_ret_v2f16:5926; GFX12: ; %bb.0:5927; GFX12-NEXT: s_wait_loadcnt_dscnt 0x05928; GFX12-NEXT: s_wait_expcnt 0x05929; GFX12-NEXT: s_wait_samplecnt 0x05930; GFX12-NEXT: s_wait_bvhcnt 0x05931; GFX12-NEXT: s_wait_kmcnt 0x05932; GFX12-NEXT: s_wait_storecnt 0x05933; GFX12-NEXT: ds_pk_add_rtn_f16 v0, v0, v15934; GFX12-NEXT: s_wait_dscnt 0x05935; GFX12-NEXT: global_inv scope:SCOPE_SE5936; GFX12-NEXT: s_wait_loadcnt 0x05937; GFX12-NEXT: s_setpc_b64 s[30:31]5938;5939; GFX942-LABEL: local_atomic_fadd_ret_v2f16:5940; GFX942: ; %bb.0:5941; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5942; GFX942-NEXT: ds_pk_add_rtn_f16 v0, v0, v15943; GFX942-NEXT: s_waitcnt lgkmcnt(0)5944; GFX942-NEXT: s_setpc_b64 s[30:31]5945;5946; GFX11-LABEL: local_atomic_fadd_ret_v2f16:5947; GFX11: ; %bb.0:5948; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5949; GFX11-NEXT: ds_load_b32 v2, v05950; GFX11-NEXT: s_mov_b32 s0, 05951; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start5952; GFX11-NEXT: ; =>This Inner Loop Header: Depth=15953; GFX11-NEXT: s_waitcnt lgkmcnt(0)5954; GFX11-NEXT: v_mov_b32_e32 v3, v25955; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)5956; GFX11-NEXT: v_pk_add_f16 v2, v3, v15957; GFX11-NEXT: s_waitcnt_vscnt null, 0x05958; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v35959; GFX11-NEXT: s_waitcnt lgkmcnt(0)5960; GFX11-NEXT: buffer_gl0_inv5961; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v35962; GFX11-NEXT: s_or_b32 s0, vcc_lo, s05963; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)5964; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s05965; GFX11-NEXT: s_cbranch_execnz .LBB20_15966; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end5967; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s05968; GFX11-NEXT: v_mov_b32_e32 v0, v25969; GFX11-NEXT: s_setpc_b64 s[30:31]5970;5971; GFX10-LABEL: local_atomic_fadd_ret_v2f16:5972; GFX10: ; %bb.0:5973; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5974; GFX10-NEXT: ds_read_b32 v2, v05975; GFX10-NEXT: s_mov_b32 s4, 05976; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start5977; GFX10-NEXT: ; =>This Inner Loop Header: Depth=15978; GFX10-NEXT: s_waitcnt lgkmcnt(0)5979; GFX10-NEXT: v_mov_b32_e32 v3, v25980; GFX10-NEXT: v_pk_add_f16 v2, v3, v15981; GFX10-NEXT: s_waitcnt_vscnt null, 0x05982; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v25983; GFX10-NEXT: s_waitcnt lgkmcnt(0)5984; GFX10-NEXT: buffer_gl0_inv5985; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v35986; GFX10-NEXT: s_or_b32 s4, vcc_lo, s45987; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s45988; GFX10-NEXT: s_cbranch_execnz .LBB20_15989; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end5990; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s45991; GFX10-NEXT: v_mov_b32_e32 v0, v25992; GFX10-NEXT: s_setpc_b64 s[30:31]5993;5994; GFX90A-LABEL: local_atomic_fadd_ret_v2f16:5995; GFX90A: ; %bb.0:5996; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)5997; GFX90A-NEXT: ds_read_b32 v3, v05998; GFX90A-NEXT: s_mov_b64 s[4:5], 05999; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start6000; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=16001; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6002; GFX90A-NEXT: v_pk_add_f16 v2, v3, v16003; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v26004; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6005; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36006; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6007; GFX90A-NEXT: v_mov_b32_e32 v3, v26008; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]6009; GFX90A-NEXT: s_cbranch_execnz .LBB20_16010; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end6011; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6012; GFX90A-NEXT: v_mov_b32_e32 v0, v26013; GFX90A-NEXT: s_setpc_b64 s[30:31]6014;6015; GFX908-LABEL: local_atomic_fadd_ret_v2f16:6016; GFX908: ; %bb.0:6017; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6018; GFX908-NEXT: ds_read_b32 v2, v06019; GFX908-NEXT: s_mov_b64 s[4:5], 06020; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start6021; GFX908-NEXT: ; =>This Inner Loop Header: Depth=16022; GFX908-NEXT: s_waitcnt lgkmcnt(0)6023; GFX908-NEXT: v_mov_b32_e32 v3, v26024; GFX908-NEXT: v_pk_add_f16 v2, v3, v16025; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v26026; GFX908-NEXT: s_waitcnt lgkmcnt(0)6027; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36028; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6029; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]6030; GFX908-NEXT: s_cbranch_execnz .LBB20_16031; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end6032; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]6033; GFX908-NEXT: v_mov_b32_e32 v0, v26034; GFX908-NEXT: s_setpc_b64 s[30:31]6035;6036; GFX8-LABEL: local_atomic_fadd_ret_v2f16:6037; GFX8: ; %bb.0:6038; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6039; GFX8-NEXT: s_mov_b32 m0, -16040; GFX8-NEXT: ds_read_b32 v2, v06041; GFX8-NEXT: s_mov_b64 s[4:5], 06042; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start6043; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16044; GFX8-NEXT: s_waitcnt lgkmcnt(0)6045; GFX8-NEXT: v_mov_b32_e32 v3, v26046; GFX8-NEXT: v_add_f16_sdwa v2, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_16047; GFX8-NEXT: v_add_f16_e32 v4, v3, v16048; GFX8-NEXT: v_or_b32_e32 v2, v4, v26049; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v26050; GFX8-NEXT: s_waitcnt lgkmcnt(0)6051; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36052; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6053; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]6054; GFX8-NEXT: s_cbranch_execnz .LBB20_16055; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6056; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]6057; GFX8-NEXT: v_mov_b32_e32 v0, v26058; GFX8-NEXT: s_setpc_b64 s[30:31]6059;6060; GFX7-LABEL: local_atomic_fadd_ret_v2f16:6061; GFX7: ; %bb.0:6062; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6063; GFX7-NEXT: s_mov_b32 m0, -16064; GFX7-NEXT: ds_read_b32 v3, v06065; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v26066; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v16067; GFX7-NEXT: s_mov_b64 s[4:5], 06068; GFX7-NEXT: s_waitcnt lgkmcnt(0)6069; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v36070; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v26071; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v36072; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v46073; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v56074; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start6075; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16076; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v16077; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v26078; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v16079; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v26080; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v16081; GFX7-NEXT: v_or_b32_e32 v7, v2, v16082; GFX7-NEXT: v_add_f32_e32 v5, v5, v36083; GFX7-NEXT: v_add_f32_e32 v6, v6, v46084; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v56085; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v66086; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v56087; GFX7-NEXT: v_or_b32_e32 v1, v6, v16088; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v16089; GFX7-NEXT: s_waitcnt lgkmcnt(0)6090; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v56091; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v56092; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v16093; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v76094; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6095; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]6096; GFX7-NEXT: s_cbranch_execnz .LBB20_16097; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6098; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]6099; GFX7-NEXT: v_mov_b32_e32 v0, v26100; GFX7-NEXT: s_setpc_b64 s[30:31]6101;6102; GFX6-LABEL: local_atomic_fadd_ret_v2f16:6103; GFX6: ; %bb.0:6104; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6105; GFX6-NEXT: s_mov_b32 m0, -16106; GFX6-NEXT: ds_read_b32 v3, v06107; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v26108; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v16109; GFX6-NEXT: s_mov_b64 s[4:5], 06110; GFX6-NEXT: s_waitcnt lgkmcnt(0)6111; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v36112; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v26113; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v36114; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v46115; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v56116; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start6117; GFX6-NEXT: ; =>This Inner Loop Header: Depth=16118; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v16119; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v26120; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v16121; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v26122; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v16123; GFX6-NEXT: v_or_b32_e32 v7, v2, v16124; GFX6-NEXT: v_add_f32_e32 v5, v5, v36125; GFX6-NEXT: v_add_f32_e32 v6, v6, v46126; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v56127; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v66128; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v56129; GFX6-NEXT: v_or_b32_e32 v1, v6, v16130; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v16131; GFX6-NEXT: s_waitcnt lgkmcnt(0)6132; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v56133; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v56134; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v16135; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v76136; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6137; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]6138; GFX6-NEXT: s_cbranch_execnz .LBB20_16139; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end6140; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]6141; GFX6-NEXT: v_mov_b32_e32 v0, v26142; GFX6-NEXT: s_setpc_b64 s[30:31]6143 %gep = getelementptr <2 x half>, ptr addrspace(3) %ptr, i32 163836144 %result = atomicrmw fadd ptr addrspace(3) %ptr, <2 x half> %val seq_cst6145 ret <2 x half> %result6146}6147 6148define <2 x half> @local_atomic_fadd_ret_v2f16__offset(ptr addrspace(3) %ptr, <2 x half> %val) {6149; GFX12-LABEL: local_atomic_fadd_ret_v2f16__offset:6150; GFX12: ; %bb.0:6151; GFX12-NEXT: s_wait_loadcnt_dscnt 0x06152; GFX12-NEXT: s_wait_expcnt 0x06153; GFX12-NEXT: s_wait_samplecnt 0x06154; GFX12-NEXT: s_wait_bvhcnt 0x06155; GFX12-NEXT: s_wait_kmcnt 0x06156; GFX12-NEXT: s_wait_storecnt 0x06157; GFX12-NEXT: ds_pk_add_rtn_f16 v0, v0, v1 offset:655326158; GFX12-NEXT: s_wait_dscnt 0x06159; GFX12-NEXT: global_inv scope:SCOPE_SE6160; GFX12-NEXT: s_wait_loadcnt 0x06161; GFX12-NEXT: s_setpc_b64 s[30:31]6162;6163; GFX942-LABEL: local_atomic_fadd_ret_v2f16__offset:6164; GFX942: ; %bb.0:6165; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6166; GFX942-NEXT: ds_pk_add_rtn_f16 v0, v0, v1 offset:655326167; GFX942-NEXT: s_waitcnt lgkmcnt(0)6168; GFX942-NEXT: s_setpc_b64 s[30:31]6169;6170; GFX11-LABEL: local_atomic_fadd_ret_v2f16__offset:6171; GFX11: ; %bb.0:6172; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6173; GFX11-NEXT: ds_load_b32 v2, v0 offset:655326174; GFX11-NEXT: s_mov_b32 s0, 06175; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start6176; GFX11-NEXT: ; =>This Inner Loop Header: Depth=16177; GFX11-NEXT: s_waitcnt lgkmcnt(0)6178; GFX11-NEXT: v_mov_b32_e32 v3, v26179; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)6180; GFX11-NEXT: v_pk_add_f16 v2, v3, v16181; GFX11-NEXT: s_waitcnt_vscnt null, 0x06182; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:655326183; GFX11-NEXT: s_waitcnt lgkmcnt(0)6184; GFX11-NEXT: buffer_gl0_inv6185; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v36186; GFX11-NEXT: s_or_b32 s0, vcc_lo, s06187; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6188; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s06189; GFX11-NEXT: s_cbranch_execnz .LBB21_16190; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end6191; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s06192; GFX11-NEXT: v_mov_b32_e32 v0, v26193; GFX11-NEXT: s_setpc_b64 s[30:31]6194;6195; GFX10-LABEL: local_atomic_fadd_ret_v2f16__offset:6196; GFX10: ; %bb.0:6197; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6198; GFX10-NEXT: ds_read_b32 v2, v0 offset:655326199; GFX10-NEXT: s_mov_b32 s4, 06200; GFX10-NEXT: .LBB21_1: ; %atomicrmw.start6201; GFX10-NEXT: ; =>This Inner Loop Header: Depth=16202; GFX10-NEXT: s_waitcnt lgkmcnt(0)6203; GFX10-NEXT: v_mov_b32_e32 v3, v26204; GFX10-NEXT: v_pk_add_f16 v2, v3, v16205; GFX10-NEXT: s_waitcnt_vscnt null, 0x06206; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:655326207; GFX10-NEXT: s_waitcnt lgkmcnt(0)6208; GFX10-NEXT: buffer_gl0_inv6209; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v36210; GFX10-NEXT: s_or_b32 s4, vcc_lo, s46211; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s46212; GFX10-NEXT: s_cbranch_execnz .LBB21_16213; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end6214; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s46215; GFX10-NEXT: v_mov_b32_e32 v0, v26216; GFX10-NEXT: s_setpc_b64 s[30:31]6217;6218; GFX90A-LABEL: local_atomic_fadd_ret_v2f16__offset:6219; GFX90A: ; %bb.0:6220; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6221; GFX90A-NEXT: ds_read_b32 v3, v0 offset:655326222; GFX90A-NEXT: s_mov_b64 s[4:5], 06223; GFX90A-NEXT: .LBB21_1: ; %atomicrmw.start6224; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=16225; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6226; GFX90A-NEXT: v_pk_add_f16 v2, v3, v16227; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:655326228; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6229; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36230; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6231; GFX90A-NEXT: v_mov_b32_e32 v3, v26232; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]6233; GFX90A-NEXT: s_cbranch_execnz .LBB21_16234; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end6235; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6236; GFX90A-NEXT: v_mov_b32_e32 v0, v26237; GFX90A-NEXT: s_setpc_b64 s[30:31]6238;6239; GFX908-LABEL: local_atomic_fadd_ret_v2f16__offset:6240; GFX908: ; %bb.0:6241; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6242; GFX908-NEXT: ds_read_b32 v2, v0 offset:655326243; GFX908-NEXT: s_mov_b64 s[4:5], 06244; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start6245; GFX908-NEXT: ; =>This Inner Loop Header: Depth=16246; GFX908-NEXT: s_waitcnt lgkmcnt(0)6247; GFX908-NEXT: v_mov_b32_e32 v3, v26248; GFX908-NEXT: v_pk_add_f16 v2, v3, v16249; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:655326250; GFX908-NEXT: s_waitcnt lgkmcnt(0)6251; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36252; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6253; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]6254; GFX908-NEXT: s_cbranch_execnz .LBB21_16255; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end6256; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]6257; GFX908-NEXT: v_mov_b32_e32 v0, v26258; GFX908-NEXT: s_setpc_b64 s[30:31]6259;6260; GFX8-LABEL: local_atomic_fadd_ret_v2f16__offset:6261; GFX8: ; %bb.0:6262; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6263; GFX8-NEXT: s_mov_b32 m0, -16264; GFX8-NEXT: ds_read_b32 v2, v0 offset:655326265; GFX8-NEXT: s_mov_b64 s[4:5], 06266; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start6267; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16268; GFX8-NEXT: s_waitcnt lgkmcnt(0)6269; GFX8-NEXT: v_mov_b32_e32 v3, v26270; GFX8-NEXT: v_add_f16_sdwa v2, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_16271; GFX8-NEXT: v_add_f16_e32 v4, v3, v16272; GFX8-NEXT: v_or_b32_e32 v2, v4, v26273; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:655326274; GFX8-NEXT: s_waitcnt lgkmcnt(0)6275; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v36276; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6277; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]6278; GFX8-NEXT: s_cbranch_execnz .LBB21_16279; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6280; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]6281; GFX8-NEXT: v_mov_b32_e32 v0, v26282; GFX8-NEXT: s_setpc_b64 s[30:31]6283;6284; GFX7-LABEL: local_atomic_fadd_ret_v2f16__offset:6285; GFX7: ; %bb.0:6286; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6287; GFX7-NEXT: s_mov_b32 m0, -16288; GFX7-NEXT: ds_read_b32 v3, v0 offset:655326289; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v26290; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v16291; GFX7-NEXT: s_mov_b64 s[4:5], 06292; GFX7-NEXT: s_waitcnt lgkmcnt(0)6293; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v36294; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v26295; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v36296; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v46297; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v56298; GFX7-NEXT: .LBB21_1: ; %atomicrmw.start6299; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16300; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v16301; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v26302; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v16303; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v26304; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v16305; GFX7-NEXT: v_or_b32_e32 v7, v2, v16306; GFX7-NEXT: v_add_f32_e32 v5, v5, v36307; GFX7-NEXT: v_add_f32_e32 v6, v6, v46308; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v56309; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v66310; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v56311; GFX7-NEXT: v_or_b32_e32 v1, v6, v16312; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v1 offset:655326313; GFX7-NEXT: s_waitcnt lgkmcnt(0)6314; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v56315; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v56316; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v16317; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v76318; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6319; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]6320; GFX7-NEXT: s_cbranch_execnz .LBB21_16321; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6322; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]6323; GFX7-NEXT: v_mov_b32_e32 v0, v26324; GFX7-NEXT: s_setpc_b64 s[30:31]6325;6326; GFX6-LABEL: local_atomic_fadd_ret_v2f16__offset:6327; GFX6: ; %bb.0:6328; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6329; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffc, v06330; GFX6-NEXT: s_mov_b32 m0, -16331; GFX6-NEXT: ds_read_b32 v4, v36332; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v26333; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v16334; GFX6-NEXT: s_mov_b64 s[4:5], 06335; GFX6-NEXT: s_waitcnt lgkmcnt(0)6336; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v46337; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v46338; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v16339; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v26340; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v56341; GFX6-NEXT: .LBB21_1: ; %atomicrmw.start6342; GFX6-NEXT: ; =>This Inner Loop Header: Depth=16343; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v16344; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v06345; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v16346; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v06347; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v16348; GFX6-NEXT: v_or_b32_e32 v7, v0, v16349; GFX6-NEXT: v_add_f32_e32 v5, v5, v26350; GFX6-NEXT: v_add_f32_e32 v6, v6, v46351; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v56352; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v66353; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v56354; GFX6-NEXT: v_or_b32_e32 v0, v6, v06355; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v3, v7, v06356; GFX6-NEXT: s_waitcnt lgkmcnt(0)6357; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v56358; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v56359; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v16360; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v76361; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6362; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]6363; GFX6-NEXT: s_cbranch_execnz .LBB21_16364; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end6365; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]6366; GFX6-NEXT: s_setpc_b64 s[30:31]6367 %gep = getelementptr <2 x half>, ptr addrspace(3) %ptr, i32 163836368 %result = atomicrmw fadd ptr addrspace(3) %gep, <2 x half> %val seq_cst6369 ret <2 x half> %result6370}6371 6372define void @local_atomic_fadd_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %val) {6373; GFX12-LABEL: local_atomic_fadd_noret_v2f16:6374; GFX12: ; %bb.0:6375; GFX12-NEXT: s_wait_loadcnt_dscnt 0x06376; GFX12-NEXT: s_wait_expcnt 0x06377; GFX12-NEXT: s_wait_samplecnt 0x06378; GFX12-NEXT: s_wait_bvhcnt 0x06379; GFX12-NEXT: s_wait_kmcnt 0x06380; GFX12-NEXT: s_wait_storecnt 0x06381; GFX12-NEXT: ds_pk_add_f16 v0, v16382; GFX12-NEXT: s_wait_dscnt 0x06383; GFX12-NEXT: global_inv scope:SCOPE_SE6384; GFX12-NEXT: s_wait_loadcnt 0x06385; GFX12-NEXT: s_setpc_b64 s[30:31]6386;6387; GFX942-LABEL: local_atomic_fadd_noret_v2f16:6388; GFX942: ; %bb.0:6389; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6390; GFX942-NEXT: ds_pk_add_f16 v0, v16391; GFX942-NEXT: s_waitcnt lgkmcnt(0)6392; GFX942-NEXT: s_setpc_b64 s[30:31]6393;6394; GFX11-LABEL: local_atomic_fadd_noret_v2f16:6395; GFX11: ; %bb.0:6396; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6397; GFX11-NEXT: ds_load_b32 v2, v06398; GFX11-NEXT: s_mov_b32 s0, 06399; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start6400; GFX11-NEXT: ; =>This Inner Loop Header: Depth=16401; GFX11-NEXT: s_waitcnt lgkmcnt(0)6402; GFX11-NEXT: v_pk_add_f16 v3, v2, v16403; GFX11-NEXT: s_waitcnt_vscnt null, 0x06404; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v26405; GFX11-NEXT: s_waitcnt lgkmcnt(0)6406; GFX11-NEXT: buffer_gl0_inv6407; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v26408; GFX11-NEXT: v_mov_b32_e32 v2, v36409; GFX11-NEXT: s_or_b32 s0, vcc_lo, s06410; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6411; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s06412; GFX11-NEXT: s_cbranch_execnz .LBB22_16413; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end6414; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s06415; GFX11-NEXT: s_setpc_b64 s[30:31]6416;6417; GFX10-LABEL: local_atomic_fadd_noret_v2f16:6418; GFX10: ; %bb.0:6419; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6420; GFX10-NEXT: ds_read_b32 v2, v06421; GFX10-NEXT: s_mov_b32 s4, 06422; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start6423; GFX10-NEXT: ; =>This Inner Loop Header: Depth=16424; GFX10-NEXT: s_waitcnt lgkmcnt(0)6425; GFX10-NEXT: v_pk_add_f16 v3, v2, v16426; GFX10-NEXT: s_waitcnt_vscnt null, 0x06427; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v36428; GFX10-NEXT: s_waitcnt lgkmcnt(0)6429; GFX10-NEXT: buffer_gl0_inv6430; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v26431; GFX10-NEXT: v_mov_b32_e32 v2, v36432; GFX10-NEXT: s_or_b32 s4, vcc_lo, s46433; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s46434; GFX10-NEXT: s_cbranch_execnz .LBB22_16435; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end6436; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s46437; GFX10-NEXT: s_setpc_b64 s[30:31]6438;6439; GFX90A-LABEL: local_atomic_fadd_noret_v2f16:6440; GFX90A: ; %bb.0:6441; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6442; GFX90A-NEXT: ds_read_b32 v2, v06443; GFX90A-NEXT: s_mov_b64 s[4:5], 06444; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start6445; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=16446; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6447; GFX90A-NEXT: v_pk_add_f16 v3, v2, v16448; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v36449; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6450; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v26451; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6452; GFX90A-NEXT: v_mov_b32_e32 v2, v36453; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]6454; GFX90A-NEXT: s_cbranch_execnz .LBB22_16455; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end6456; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6457; GFX90A-NEXT: s_setpc_b64 s[30:31]6458;6459; GFX908-LABEL: local_atomic_fadd_noret_v2f16:6460; GFX908: ; %bb.0:6461; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6462; GFX908-NEXT: ds_read_b32 v2, v06463; GFX908-NEXT: s_mov_b64 s[4:5], 06464; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start6465; GFX908-NEXT: ; =>This Inner Loop Header: Depth=16466; GFX908-NEXT: s_waitcnt lgkmcnt(0)6467; GFX908-NEXT: v_pk_add_f16 v3, v2, v16468; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v36469; GFX908-NEXT: s_waitcnt lgkmcnt(0)6470; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v26471; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6472; GFX908-NEXT: v_mov_b32_e32 v2, v36473; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]6474; GFX908-NEXT: s_cbranch_execnz .LBB22_16475; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end6476; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]6477; GFX908-NEXT: s_setpc_b64 s[30:31]6478;6479; GFX8-LABEL: local_atomic_fadd_noret_v2f16:6480; GFX8: ; %bb.0:6481; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6482; GFX8-NEXT: s_mov_b32 m0, -16483; GFX8-NEXT: ds_read_b32 v2, v06484; GFX8-NEXT: s_mov_b64 s[4:5], 06485; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start6486; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16487; GFX8-NEXT: s_waitcnt lgkmcnt(0)6488; GFX8-NEXT: v_add_f16_sdwa v3, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_16489; GFX8-NEXT: v_add_f16_e32 v4, v2, v16490; GFX8-NEXT: v_or_b32_e32 v3, v4, v36491; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v36492; GFX8-NEXT: s_waitcnt lgkmcnt(0)6493; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v26494; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6495; GFX8-NEXT: v_mov_b32_e32 v2, v36496; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]6497; GFX8-NEXT: s_cbranch_execnz .LBB22_16498; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6499; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]6500; GFX8-NEXT: s_setpc_b64 s[30:31]6501;6502; GFX7-LABEL: local_atomic_fadd_noret_v2f16:6503; GFX7: ; %bb.0:6504; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6505; GFX7-NEXT: s_mov_b32 m0, -16506; GFX7-NEXT: ds_read_b32 v3, v06507; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v26508; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v16509; GFX7-NEXT: s_mov_b64 s[4:5], 06510; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v26511; GFX7-NEXT: s_waitcnt lgkmcnt(0)6512; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v36513; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v46514; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v36515; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v56516; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start6517; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16518; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v46519; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v36520; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v46521; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v36522; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v46523; GFX7-NEXT: v_or_b32_e32 v7, v3, v46524; GFX7-NEXT: v_add_f32_e32 v5, v5, v16525; GFX7-NEXT: v_add_f32_e32 v6, v6, v26526; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v56527; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v66528; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v56529; GFX7-NEXT: v_or_b32_e32 v3, v6, v36530; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v36531; GFX7-NEXT: s_waitcnt lgkmcnt(0)6532; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v56533; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v56534; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v46535; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v76536; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6537; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]6538; GFX7-NEXT: s_cbranch_execnz .LBB22_16539; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6540; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]6541; GFX7-NEXT: s_setpc_b64 s[30:31]6542;6543; GFX6-LABEL: local_atomic_fadd_noret_v2f16:6544; GFX6: ; %bb.0:6545; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6546; GFX6-NEXT: s_mov_b32 m0, -16547; GFX6-NEXT: ds_read_b32 v3, v06548; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v26549; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v16550; GFX6-NEXT: s_mov_b64 s[4:5], 06551; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v26552; GFX6-NEXT: s_waitcnt lgkmcnt(0)6553; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v36554; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v46555; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v36556; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v56557; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start6558; GFX6-NEXT: ; =>This Inner Loop Header: Depth=16559; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v46560; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v36561; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v46562; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v36563; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v46564; GFX6-NEXT: v_or_b32_e32 v7, v3, v46565; GFX6-NEXT: v_add_f32_e32 v5, v5, v16566; GFX6-NEXT: v_add_f32_e32 v6, v6, v26567; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v56568; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v66569; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v56570; GFX6-NEXT: v_or_b32_e32 v3, v6, v36571; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v36572; GFX6-NEXT: s_waitcnt lgkmcnt(0)6573; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v56574; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v56575; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v46576; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v76577; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6578; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]6579; GFX6-NEXT: s_cbranch_execnz .LBB22_16580; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end6581; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]6582; GFX6-NEXT: s_setpc_b64 s[30:31]6583 %result = atomicrmw fadd ptr addrspace(3) %ptr, <2 x half> %val seq_cst6584 ret void6585}6586 6587define void @local_atomic_fadd_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x half> %val) {6588; GFX12-LABEL: local_atomic_fadd_noret_v2f16__offset:6589; GFX12: ; %bb.0:6590; GFX12-NEXT: s_wait_loadcnt_dscnt 0x06591; GFX12-NEXT: s_wait_expcnt 0x06592; GFX12-NEXT: s_wait_samplecnt 0x06593; GFX12-NEXT: s_wait_bvhcnt 0x06594; GFX12-NEXT: s_wait_kmcnt 0x06595; GFX12-NEXT: s_wait_storecnt 0x06596; GFX12-NEXT: ds_pk_add_f16 v0, v1 offset:655326597; GFX12-NEXT: s_wait_dscnt 0x06598; GFX12-NEXT: global_inv scope:SCOPE_SE6599; GFX12-NEXT: s_wait_loadcnt 0x06600; GFX12-NEXT: s_setpc_b64 s[30:31]6601;6602; GFX942-LABEL: local_atomic_fadd_noret_v2f16__offset:6603; GFX942: ; %bb.0:6604; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6605; GFX942-NEXT: ds_pk_add_f16 v0, v1 offset:655326606; GFX942-NEXT: s_waitcnt lgkmcnt(0)6607; GFX942-NEXT: s_setpc_b64 s[30:31]6608;6609; GFX11-LABEL: local_atomic_fadd_noret_v2f16__offset:6610; GFX11: ; %bb.0:6611; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6612; GFX11-NEXT: ds_load_b32 v2, v0 offset:655326613; GFX11-NEXT: s_mov_b32 s0, 06614; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start6615; GFX11-NEXT: ; =>This Inner Loop Header: Depth=16616; GFX11-NEXT: s_waitcnt lgkmcnt(0)6617; GFX11-NEXT: v_pk_add_f16 v3, v2, v16618; GFX11-NEXT: s_waitcnt_vscnt null, 0x06619; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:655326620; GFX11-NEXT: s_waitcnt lgkmcnt(0)6621; GFX11-NEXT: buffer_gl0_inv6622; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v26623; GFX11-NEXT: v_mov_b32_e32 v2, v36624; GFX11-NEXT: s_or_b32 s0, vcc_lo, s06625; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6626; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s06627; GFX11-NEXT: s_cbranch_execnz .LBB23_16628; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end6629; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s06630; GFX11-NEXT: s_setpc_b64 s[30:31]6631;6632; GFX10-LABEL: local_atomic_fadd_noret_v2f16__offset:6633; GFX10: ; %bb.0:6634; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6635; GFX10-NEXT: ds_read_b32 v2, v0 offset:655326636; GFX10-NEXT: s_mov_b32 s4, 06637; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start6638; GFX10-NEXT: ; =>This Inner Loop Header: Depth=16639; GFX10-NEXT: s_waitcnt lgkmcnt(0)6640; GFX10-NEXT: v_pk_add_f16 v3, v2, v16641; GFX10-NEXT: s_waitcnt_vscnt null, 0x06642; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:655326643; GFX10-NEXT: s_waitcnt lgkmcnt(0)6644; GFX10-NEXT: buffer_gl0_inv6645; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v26646; GFX10-NEXT: v_mov_b32_e32 v2, v36647; GFX10-NEXT: s_or_b32 s4, vcc_lo, s46648; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s46649; GFX10-NEXT: s_cbranch_execnz .LBB23_16650; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end6651; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s46652; GFX10-NEXT: s_setpc_b64 s[30:31]6653;6654; GFX90A-LABEL: local_atomic_fadd_noret_v2f16__offset:6655; GFX90A: ; %bb.0:6656; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6657; GFX90A-NEXT: ds_read_b32 v2, v0 offset:655326658; GFX90A-NEXT: s_mov_b64 s[4:5], 06659; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start6660; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=16661; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6662; GFX90A-NEXT: v_pk_add_f16 v3, v2, v16663; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:655326664; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6665; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v26666; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6667; GFX90A-NEXT: v_mov_b32_e32 v2, v36668; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]6669; GFX90A-NEXT: s_cbranch_execnz .LBB23_16670; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end6671; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]6672; GFX90A-NEXT: s_setpc_b64 s[30:31]6673;6674; GFX908-LABEL: local_atomic_fadd_noret_v2f16__offset:6675; GFX908: ; %bb.0:6676; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6677; GFX908-NEXT: ds_read_b32 v2, v0 offset:655326678; GFX908-NEXT: s_mov_b64 s[4:5], 06679; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start6680; GFX908-NEXT: ; =>This Inner Loop Header: Depth=16681; GFX908-NEXT: s_waitcnt lgkmcnt(0)6682; GFX908-NEXT: v_pk_add_f16 v3, v2, v16683; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:655326684; GFX908-NEXT: s_waitcnt lgkmcnt(0)6685; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v26686; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6687; GFX908-NEXT: v_mov_b32_e32 v2, v36688; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]6689; GFX908-NEXT: s_cbranch_execnz .LBB23_16690; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end6691; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]6692; GFX908-NEXT: s_setpc_b64 s[30:31]6693;6694; GFX8-LABEL: local_atomic_fadd_noret_v2f16__offset:6695; GFX8: ; %bb.0:6696; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6697; GFX8-NEXT: s_mov_b32 m0, -16698; GFX8-NEXT: ds_read_b32 v2, v0 offset:655326699; GFX8-NEXT: s_mov_b64 s[4:5], 06700; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start6701; GFX8-NEXT: ; =>This Inner Loop Header: Depth=16702; GFX8-NEXT: s_waitcnt lgkmcnt(0)6703; GFX8-NEXT: v_add_f16_sdwa v3, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_16704; GFX8-NEXT: v_add_f16_e32 v4, v2, v16705; GFX8-NEXT: v_or_b32_e32 v3, v4, v36706; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:655326707; GFX8-NEXT: s_waitcnt lgkmcnt(0)6708; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v26709; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6710; GFX8-NEXT: v_mov_b32_e32 v2, v36711; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]6712; GFX8-NEXT: s_cbranch_execnz .LBB23_16713; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end6714; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]6715; GFX8-NEXT: s_setpc_b64 s[30:31]6716;6717; GFX7-LABEL: local_atomic_fadd_noret_v2f16__offset:6718; GFX7: ; %bb.0:6719; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6720; GFX7-NEXT: s_mov_b32 m0, -16721; GFX7-NEXT: ds_read_b32 v3, v0 offset:655326722; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v26723; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v16724; GFX7-NEXT: s_mov_b64 s[4:5], 06725; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v26726; GFX7-NEXT: s_waitcnt lgkmcnt(0)6727; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v36728; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v46729; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v36730; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v56731; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start6732; GFX7-NEXT: ; =>This Inner Loop Header: Depth=16733; GFX7-NEXT: v_cvt_f16_f32_e32 v4, v46734; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v36735; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v46736; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v36737; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v46738; GFX7-NEXT: v_or_b32_e32 v7, v3, v46739; GFX7-NEXT: v_add_f32_e32 v5, v5, v16740; GFX7-NEXT: v_add_f32_e32 v6, v6, v26741; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v56742; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v66743; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v56744; GFX7-NEXT: v_or_b32_e32 v3, v6, v36745; GFX7-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v3 offset:655326746; GFX7-NEXT: s_waitcnt lgkmcnt(0)6747; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v56748; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v56749; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v46750; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v76751; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6752; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]6753; GFX7-NEXT: s_cbranch_execnz .LBB23_16754; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end6755; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]6756; GFX7-NEXT: s_setpc_b64 s[30:31]6757;6758; GFX6-LABEL: local_atomic_fadd_noret_v2f16__offset:6759; GFX6: ; %bb.0:6760; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6761; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v06762; GFX6-NEXT: s_mov_b32 m0, -16763; GFX6-NEXT: ds_read_b32 v4, v06764; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v26765; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v16766; GFX6-NEXT: s_mov_b64 s[4:5], 06767; GFX6-NEXT: s_waitcnt lgkmcnt(0)6768; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v46769; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v46770; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v16771; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v26772; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v56773; GFX6-NEXT: .LBB23_1: ; %atomicrmw.start6774; GFX6-NEXT: ; =>This Inner Loop Header: Depth=16775; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v46776; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v36777; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v46778; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v36779; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v46780; GFX6-NEXT: v_or_b32_e32 v7, v3, v46781; GFX6-NEXT: v_add_f32_e32 v5, v5, v16782; GFX6-NEXT: v_add_f32_e32 v6, v6, v26783; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v56784; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v66785; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v56786; GFX6-NEXT: v_or_b32_e32 v3, v6, v36787; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v7, v36788; GFX6-NEXT: s_waitcnt lgkmcnt(0)6789; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v56790; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v56791; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v46792; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v76793; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]6794; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]6795; GFX6-NEXT: s_cbranch_execnz .LBB23_16796; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end6797; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]6798; GFX6-NEXT: s_setpc_b64 s[30:31]6799 %gep = getelementptr <2 x half>, ptr addrspace(3) %ptr, i32 163836800 %result = atomicrmw fadd ptr addrspace(3) %gep, <2 x half> %val seq_cst6801 ret void6802}6803 6804; --------------------------------------------------------------------6805; <2 x bfloat>6806; --------------------------------------------------------------------6807 6808define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat> %val) {6809; GFX12-LABEL: local_atomic_fadd_ret_v2bf16:6810; GFX12: ; %bb.0:6811; GFX12-NEXT: s_wait_loadcnt_dscnt 0x06812; GFX12-NEXT: s_wait_expcnt 0x06813; GFX12-NEXT: s_wait_samplecnt 0x06814; GFX12-NEXT: s_wait_bvhcnt 0x06815; GFX12-NEXT: s_wait_kmcnt 0x06816; GFX12-NEXT: s_wait_storecnt 0x06817; GFX12-NEXT: ds_pk_add_rtn_bf16 v0, v0, v16818; GFX12-NEXT: s_wait_dscnt 0x06819; GFX12-NEXT: global_inv scope:SCOPE_SE6820; GFX12-NEXT: s_wait_loadcnt 0x06821; GFX12-NEXT: s_setpc_b64 s[30:31]6822;6823; GFX942-LABEL: local_atomic_fadd_ret_v2bf16:6824; GFX942: ; %bb.0:6825; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6826; GFX942-NEXT: ds_pk_add_rtn_bf16 v0, v0, v16827; GFX942-NEXT: s_waitcnt lgkmcnt(0)6828; GFX942-NEXT: s_setpc_b64 s[30:31]6829;6830; GFX11-TRUE16-LABEL: local_atomic_fadd_ret_v2bf16:6831; GFX11-TRUE16: ; %bb.0:6832; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6833; GFX11-TRUE16-NEXT: ds_load_b32 v2, v06834; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v16835; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v16836; GFX11-TRUE16-NEXT: s_mov_b32 s0, 06837; GFX11-TRUE16-NEXT: .p2align 66838; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start6839; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=16840; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)6841; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v26842; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6843; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v46844; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v36845; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)6846; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 16847; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v56848; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff6849; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v46850; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6851; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v16852; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 16853; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v26854; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v26855; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)6856; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff6857; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo6858; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v56859; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo6860; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)6861; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h6862; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x06863; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v46864; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)6865; GFX11-TRUE16-NEXT: buffer_gl0_inv6866; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v46867; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s06868; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6869; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s06870; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB24_16871; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end6872; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s06873; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v26874; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]6875;6876; GFX11-FAKE16-LABEL: local_atomic_fadd_ret_v2bf16:6877; GFX11-FAKE16: ; %bb.0:6878; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6879; GFX11-FAKE16-NEXT: ds_load_b32 v2, v06880; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v16881; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v16882; GFX11-FAKE16-NEXT: s_mov_b32 s1, 06883; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x16884; GFX11-FAKE16-NEXT: .p2align 66885; GFX11-FAKE16-NEXT: .LBB24_1: ; %atomicrmw.start6886; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=16887; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)6888; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v26889; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6890; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v46891; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v16892; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)6893; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 16894; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v56895; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v56896; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff6897; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6898; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v46899; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v36900; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)6901; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 16902; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v26903; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v26904; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff6905; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)6906; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s06907; GFX11-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x70603026908; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x06909; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v46910; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)6911; GFX11-FAKE16-NEXT: buffer_gl0_inv6912; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v46913; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s16914; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)6915; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s16916; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB24_16917; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end6918; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x26919; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s16920; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v26921; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]6922;6923; GFX10-LABEL: local_atomic_fadd_ret_v2bf16:6924; GFX10: ; %bb.0:6925; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6926; GFX10-NEXT: ds_read_b32 v2, v06927; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v16928; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v16929; GFX10-NEXT: s_mov_b32 s5, 06930; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start6931; GFX10-NEXT: ; =>This Inner Loop Header: Depth=16932; GFX10-NEXT: s_waitcnt lgkmcnt(0)6933; GFX10-NEXT: v_mov_b32_e32 v4, v26934; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v46935; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v46936; GFX10-NEXT: v_add_f32_e32 v2, v2, v36937; GFX10-NEXT: v_add_f32_e32 v5, v5, v16938; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 16939; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 16940; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v26941; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v56942; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v56943; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff6944; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff6945; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v26946; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo6947; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s46948; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x70603026949; GFX10-NEXT: s_waitcnt_vscnt null, 0x06950; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v26951; GFX10-NEXT: s_waitcnt lgkmcnt(0)6952; GFX10-NEXT: buffer_gl0_inv6953; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v46954; GFX10-NEXT: s_or_b32 s5, vcc_lo, s56955; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s56956; GFX10-NEXT: s_cbranch_execnz .LBB24_16957; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end6958; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s56959; GFX10-NEXT: v_mov_b32_e32 v0, v26960; GFX10-NEXT: s_setpc_b64 s[30:31]6961;6962; GFX90A-LABEL: local_atomic_fadd_ret_v2bf16:6963; GFX90A: ; %bb.0:6964; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)6965; GFX90A-NEXT: ds_read_b32 v4, v06966; GFX90A-NEXT: s_mov_b64 s[6:7], 06967; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v16968; GFX90A-NEXT: s_movk_i32 s8, 0x7fff6969; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v16970; GFX90A-NEXT: s_mov_b32 s9, 0x70603026971; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start6972; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=16973; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6974; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v46975; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v46976; GFX90A-NEXT: v_add_f32_e32 v1, v1, v26977; GFX90A-NEXT: v_add_f32_e32 v5, v5, v36978; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 16979; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 16980; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v16981; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v56982; GFX90A-NEXT: v_add3_u32 v6, v6, v1, s86983; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s86984; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v56985; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v16986; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]6987; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc6988; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s96989; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v16990; GFX90A-NEXT: s_waitcnt lgkmcnt(0)6991; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v46992; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]6993; GFX90A-NEXT: v_mov_b32_e32 v4, v16994; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]6995; GFX90A-NEXT: s_cbranch_execnz .LBB24_16996; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end6997; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]6998; GFX90A-NEXT: v_mov_b32_e32 v0, v16999; GFX90A-NEXT: s_setpc_b64 s[30:31]7000;7001; GFX908-LABEL: local_atomic_fadd_ret_v2bf16:7002; GFX908: ; %bb.0:7003; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7004; GFX908-NEXT: ds_read_b32 v2, v07005; GFX908-NEXT: s_mov_b64 s[6:7], 07006; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v17007; GFX908-NEXT: s_movk_i32 s8, 0x7fff7008; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v17009; GFX908-NEXT: s_mov_b32 s9, 0x70603027010; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start7011; GFX908-NEXT: ; =>This Inner Loop Header: Depth=17012; GFX908-NEXT: s_waitcnt lgkmcnt(0)7013; GFX908-NEXT: v_mov_b32_e32 v4, v27014; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v47015; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v47016; GFX908-NEXT: v_add_f32_e32 v2, v2, v37017; GFX908-NEXT: v_add_f32_e32 v5, v5, v17018; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 17019; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 17020; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v27021; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v57022; GFX908-NEXT: v_add3_u32 v6, v6, v2, s87023; GFX908-NEXT: v_add3_u32 v8, v8, v5, s87024; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v57025; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v27026; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]7027; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc7028; GFX908-NEXT: v_perm_b32 v2, v5, v2, s97029; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v27030; GFX908-NEXT: s_waitcnt lgkmcnt(0)7031; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v47032; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7033; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]7034; GFX908-NEXT: s_cbranch_execnz .LBB24_17035; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end7036; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]7037; GFX908-NEXT: v_mov_b32_e32 v0, v27038; GFX908-NEXT: s_setpc_b64 s[30:31]7039;7040; GFX8-LABEL: local_atomic_fadd_ret_v2bf16:7041; GFX8: ; %bb.0:7042; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7043; GFX8-NEXT: s_mov_b32 m0, -17044; GFX8-NEXT: ds_read_b32 v2, v07045; GFX8-NEXT: s_mov_b64 s[6:7], 07046; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v17047; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v17048; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start7049; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17050; GFX8-NEXT: s_waitcnt lgkmcnt(0)7051; GFX8-NEXT: v_mov_b32_e32 v4, v27052; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v47053; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v47054; GFX8-NEXT: v_add_f32_e32 v2, v2, v37055; GFX8-NEXT: v_add_f32_e32 v5, v5, v17056; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 17057; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 17058; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v27059; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v57060; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v67061; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v87062; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v57063; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v57064; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v27065; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v27066; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc7067; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]7068; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v57069; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 167070; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v27071; GFX8-NEXT: s_waitcnt lgkmcnt(0)7072; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v47073; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7074; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]7075; GFX8-NEXT: s_cbranch_execnz .LBB24_17076; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7077; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]7078; GFX8-NEXT: v_mov_b32_e32 v0, v27079; GFX8-NEXT: s_setpc_b64 s[30:31]7080;7081; GFX7-LABEL: local_atomic_fadd_ret_v2bf16:7082; GFX7: ; %bb.0:7083; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7084; GFX7-NEXT: s_mov_b32 m0, -17085; GFX7-NEXT: ds_read_b32 v3, v07086; GFX7-NEXT: v_mov_b32_e32 v4, v17087; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v47088; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v27089; GFX7-NEXT: s_mov_b64 s[4:5], 07090; GFX7-NEXT: s_waitcnt lgkmcnt(0)7091; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v37092; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v37093; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v47094; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v27095; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start7096; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17097; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v17098; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v37099; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v17100; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v37101; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v17102; GFX7-NEXT: v_add_f32_e32 v6, v6, v27103; GFX7-NEXT: v_add_f32_e32 v5, v5, v47104; GFX7-NEXT: v_alignbit_b32 v1, v1, v3, 167105; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v67106; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 167107; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v1, v37108; GFX7-NEXT: s_waitcnt lgkmcnt(0)7109; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v17110; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v37111; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7112; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v37113; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]7114; GFX7-NEXT: s_cbranch_execnz .LBB24_17115; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7116; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]7117; GFX7-NEXT: v_mov_b32_e32 v0, v37118; GFX7-NEXT: s_setpc_b64 s[30:31]7119;7120; GFX6-LABEL: local_atomic_fadd_ret_v2bf16:7121; GFX6: ; %bb.0:7122; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7123; GFX6-NEXT: s_mov_b32 m0, -17124; GFX6-NEXT: ds_read_b32 v3, v07125; GFX6-NEXT: v_mov_b32_e32 v4, v17126; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v47127; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v27128; GFX6-NEXT: s_mov_b64 s[4:5], 07129; GFX6-NEXT: s_waitcnt lgkmcnt(0)7130; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v37131; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v37132; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v47133; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v27134; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start7135; GFX6-NEXT: ; =>This Inner Loop Header: Depth=17136; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v17137; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v37138; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v17139; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v37140; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v17141; GFX6-NEXT: v_add_f32_e32 v6, v6, v27142; GFX6-NEXT: v_add_f32_e32 v5, v5, v47143; GFX6-NEXT: v_alignbit_b32 v1, v1, v3, 167144; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v67145; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 167146; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v1, v37147; GFX6-NEXT: s_waitcnt lgkmcnt(0)7148; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v17149; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v37150; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7151; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v37152; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]7153; GFX6-NEXT: s_cbranch_execnz .LBB24_17154; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end7155; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]7156; GFX6-NEXT: v_mov_b32_e32 v0, v37157; GFX6-NEXT: s_setpc_b64 s[30:31]7158 %result = atomicrmw fadd ptr addrspace(3) %ptr, <2 x bfloat> %val seq_cst7159 ret <2 x bfloat> %result7160}7161 7162define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr, <2 x bfloat> %val) {7163; GFX12-LABEL: local_atomic_fadd_ret_v2bf16__offset:7164; GFX12: ; %bb.0:7165; GFX12-NEXT: s_wait_loadcnt_dscnt 0x07166; GFX12-NEXT: s_wait_expcnt 0x07167; GFX12-NEXT: s_wait_samplecnt 0x07168; GFX12-NEXT: s_wait_bvhcnt 0x07169; GFX12-NEXT: s_wait_kmcnt 0x07170; GFX12-NEXT: s_wait_storecnt 0x07171; GFX12-NEXT: ds_pk_add_rtn_bf16 v0, v0, v1 offset:655327172; GFX12-NEXT: s_wait_dscnt 0x07173; GFX12-NEXT: global_inv scope:SCOPE_SE7174; GFX12-NEXT: s_wait_loadcnt 0x07175; GFX12-NEXT: s_setpc_b64 s[30:31]7176;7177; GFX942-LABEL: local_atomic_fadd_ret_v2bf16__offset:7178; GFX942: ; %bb.0:7179; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7180; GFX942-NEXT: ds_pk_add_rtn_bf16 v0, v0, v1 offset:655327181; GFX942-NEXT: s_waitcnt lgkmcnt(0)7182; GFX942-NEXT: s_setpc_b64 s[30:31]7183;7184; GFX11-TRUE16-LABEL: local_atomic_fadd_ret_v2bf16__offset:7185; GFX11-TRUE16: ; %bb.0:7186; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7187; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0 offset:655327188; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v17189; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v17190; GFX11-TRUE16-NEXT: s_mov_b32 s0, 07191; GFX11-TRUE16-NEXT: .p2align 67192; GFX11-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start7193; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=17194; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)7195; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v27196; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7197; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v47198; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v37199; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)7200; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 17201; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v57202; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff7203; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v47204; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7205; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v17206; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 17207; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v27208; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v27209; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)7210; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff7211; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo7212; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v57213; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo7214; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)7215; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h7216; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x07217; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:655327218; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)7219; GFX11-TRUE16-NEXT: buffer_gl0_inv7220; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v47221; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s07222; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7223; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s07224; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB25_17225; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end7226; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s07227; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v27228; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]7229;7230; GFX11-FAKE16-LABEL: local_atomic_fadd_ret_v2bf16__offset:7231; GFX11-FAKE16: ; %bb.0:7232; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7233; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0 offset:655327234; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v17235; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v17236; GFX11-FAKE16-NEXT: s_mov_b32 s1, 07237; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x17238; GFX11-FAKE16-NEXT: .p2align 67239; GFX11-FAKE16-NEXT: .LBB25_1: ; %atomicrmw.start7240; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=17241; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)7242; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v27243; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7244; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v47245; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v17246; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)7247; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 17248; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v57249; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v57250; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff7251; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7252; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v47253; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v37254; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)7255; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 17256; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v27257; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v27258; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff7259; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7260; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s07261; GFX11-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x70603027262; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x07263; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:655327264; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)7265; GFX11-FAKE16-NEXT: buffer_gl0_inv7266; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v47267; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s17268; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7269; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s17270; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB25_17271; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end7272; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x27273; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s17274; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v27275; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]7276;7277; GFX10-LABEL: local_atomic_fadd_ret_v2bf16__offset:7278; GFX10: ; %bb.0:7279; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7280; GFX10-NEXT: ds_read_b32 v2, v0 offset:655327281; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v17282; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v17283; GFX10-NEXT: s_mov_b32 s5, 07284; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start7285; GFX10-NEXT: ; =>This Inner Loop Header: Depth=17286; GFX10-NEXT: s_waitcnt lgkmcnt(0)7287; GFX10-NEXT: v_mov_b32_e32 v4, v27288; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v47289; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v47290; GFX10-NEXT: v_add_f32_e32 v2, v2, v37291; GFX10-NEXT: v_add_f32_e32 v5, v5, v17292; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 17293; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 17294; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v27295; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v57296; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v57297; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff7298; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff7299; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v27300; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo7301; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s47302; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x70603027303; GFX10-NEXT: s_waitcnt_vscnt null, 0x07304; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:655327305; GFX10-NEXT: s_waitcnt lgkmcnt(0)7306; GFX10-NEXT: buffer_gl0_inv7307; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v47308; GFX10-NEXT: s_or_b32 s5, vcc_lo, s57309; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s57310; GFX10-NEXT: s_cbranch_execnz .LBB25_17311; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end7312; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s57313; GFX10-NEXT: v_mov_b32_e32 v0, v27314; GFX10-NEXT: s_setpc_b64 s[30:31]7315;7316; GFX90A-LABEL: local_atomic_fadd_ret_v2bf16__offset:7317; GFX90A: ; %bb.0:7318; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7319; GFX90A-NEXT: ds_read_b32 v4, v0 offset:655327320; GFX90A-NEXT: s_mov_b64 s[6:7], 07321; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v17322; GFX90A-NEXT: s_movk_i32 s8, 0x7fff7323; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v17324; GFX90A-NEXT: s_mov_b32 s9, 0x70603027325; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start7326; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=17327; GFX90A-NEXT: s_waitcnt lgkmcnt(0)7328; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v47329; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v47330; GFX90A-NEXT: v_add_f32_e32 v1, v1, v27331; GFX90A-NEXT: v_add_f32_e32 v5, v5, v37332; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 17333; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 17334; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v17335; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v57336; GFX90A-NEXT: v_add3_u32 v6, v6, v1, s87337; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s87338; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v57339; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v17340; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]7341; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc7342; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s97343; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:655327344; GFX90A-NEXT: s_waitcnt lgkmcnt(0)7345; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v47346; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7347; GFX90A-NEXT: v_mov_b32_e32 v4, v17348; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]7349; GFX90A-NEXT: s_cbranch_execnz .LBB25_17350; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end7351; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]7352; GFX90A-NEXT: v_mov_b32_e32 v0, v17353; GFX90A-NEXT: s_setpc_b64 s[30:31]7354;7355; GFX908-LABEL: local_atomic_fadd_ret_v2bf16__offset:7356; GFX908: ; %bb.0:7357; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7358; GFX908-NEXT: ds_read_b32 v2, v0 offset:655327359; GFX908-NEXT: s_mov_b64 s[6:7], 07360; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v17361; GFX908-NEXT: s_movk_i32 s8, 0x7fff7362; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v17363; GFX908-NEXT: s_mov_b32 s9, 0x70603027364; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start7365; GFX908-NEXT: ; =>This Inner Loop Header: Depth=17366; GFX908-NEXT: s_waitcnt lgkmcnt(0)7367; GFX908-NEXT: v_mov_b32_e32 v4, v27368; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v47369; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v47370; GFX908-NEXT: v_add_f32_e32 v2, v2, v37371; GFX908-NEXT: v_add_f32_e32 v5, v5, v17372; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 17373; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 17374; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v27375; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v57376; GFX908-NEXT: v_add3_u32 v6, v6, v2, s87377; GFX908-NEXT: v_add3_u32 v8, v8, v5, s87378; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v57379; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v27380; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]7381; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc7382; GFX908-NEXT: v_perm_b32 v2, v5, v2, s97383; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:655327384; GFX908-NEXT: s_waitcnt lgkmcnt(0)7385; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v47386; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7387; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]7388; GFX908-NEXT: s_cbranch_execnz .LBB25_17389; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end7390; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]7391; GFX908-NEXT: v_mov_b32_e32 v0, v27392; GFX908-NEXT: s_setpc_b64 s[30:31]7393;7394; GFX8-LABEL: local_atomic_fadd_ret_v2bf16__offset:7395; GFX8: ; %bb.0:7396; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7397; GFX8-NEXT: s_mov_b32 m0, -17398; GFX8-NEXT: ds_read_b32 v2, v0 offset:655327399; GFX8-NEXT: s_mov_b64 s[6:7], 07400; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v17401; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v17402; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start7403; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17404; GFX8-NEXT: s_waitcnt lgkmcnt(0)7405; GFX8-NEXT: v_mov_b32_e32 v4, v27406; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v47407; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v47408; GFX8-NEXT: v_add_f32_e32 v2, v2, v37409; GFX8-NEXT: v_add_f32_e32 v5, v5, v17410; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 17411; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 17412; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v27413; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v57414; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v67415; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v87416; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v57417; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v57418; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v27419; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v27420; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc7421; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]7422; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v57423; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 167424; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:655327425; GFX8-NEXT: s_waitcnt lgkmcnt(0)7426; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v47427; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7428; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]7429; GFX8-NEXT: s_cbranch_execnz .LBB25_17430; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7431; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]7432; GFX8-NEXT: v_mov_b32_e32 v0, v27433; GFX8-NEXT: s_setpc_b64 s[30:31]7434;7435; GFX7-LABEL: local_atomic_fadd_ret_v2bf16__offset:7436; GFX7: ; %bb.0:7437; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7438; GFX7-NEXT: s_mov_b32 m0, -17439; GFX7-NEXT: ds_read_b32 v3, v0 offset:655327440; GFX7-NEXT: v_mov_b32_e32 v4, v17441; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v47442; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v27443; GFX7-NEXT: s_mov_b64 s[4:5], 07444; GFX7-NEXT: s_waitcnt lgkmcnt(0)7445; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v37446; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v37447; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v47448; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v27449; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start7450; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17451; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v17452; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v37453; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v17454; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v37455; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v17456; GFX7-NEXT: v_add_f32_e32 v6, v6, v27457; GFX7-NEXT: v_add_f32_e32 v5, v5, v47458; GFX7-NEXT: v_alignbit_b32 v1, v1, v3, 167459; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v67460; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 167461; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v0, v1, v3 offset:655327462; GFX7-NEXT: s_waitcnt lgkmcnt(0)7463; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v17464; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v37465; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7466; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v37467; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]7468; GFX7-NEXT: s_cbranch_execnz .LBB25_17469; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7470; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]7471; GFX7-NEXT: v_mov_b32_e32 v0, v37472; GFX7-NEXT: s_setpc_b64 s[30:31]7473;7474; GFX6-LABEL: local_atomic_fadd_ret_v2bf16__offset:7475; GFX6: ; %bb.0:7476; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7477; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffc, v07478; GFX6-NEXT: s_mov_b32 m0, -17479; GFX6-NEXT: ds_read_b32 v0, v47480; GFX6-NEXT: v_mov_b32_e32 v3, v17481; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v37482; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v27483; GFX6-NEXT: s_mov_b64 s[4:5], 07484; GFX6-NEXT: s_waitcnt lgkmcnt(0)7485; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v07486; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v07487; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v37488; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v27489; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start7490; GFX6-NEXT: ; =>This Inner Loop Header: Depth=17491; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v17492; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v07493; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v17494; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v07495; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v17496; GFX6-NEXT: v_add_f32_e32 v6, v6, v27497; GFX6-NEXT: v_add_f32_e32 v5, v5, v37498; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 167499; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v67500; GFX6-NEXT: v_alignbit_b32 v1, v1, v5, 167501; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v4, v0, v17502; GFX6-NEXT: s_waitcnt lgkmcnt(0)7503; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v07504; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v57505; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7506; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v57507; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]7508; GFX6-NEXT: s_cbranch_execnz .LBB25_17509; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end7510; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]7511; GFX6-NEXT: s_setpc_b64 s[30:31]7512 %gep = getelementptr <2 x bfloat>, ptr addrspace(3) %ptr, i32 163837513 %result = atomicrmw fadd ptr addrspace(3) %gep, <2 x bfloat> %val seq_cst7514 ret <2 x bfloat> %result7515}7516 7517define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat> %val) {7518; GFX12-LABEL: local_atomic_fadd_noret_v2bf16:7519; GFX12: ; %bb.0:7520; GFX12-NEXT: s_wait_loadcnt_dscnt 0x07521; GFX12-NEXT: s_wait_expcnt 0x07522; GFX12-NEXT: s_wait_samplecnt 0x07523; GFX12-NEXT: s_wait_bvhcnt 0x07524; GFX12-NEXT: s_wait_kmcnt 0x07525; GFX12-NEXT: s_wait_storecnt 0x07526; GFX12-NEXT: ds_pk_add_bf16 v0, v17527; GFX12-NEXT: s_wait_dscnt 0x07528; GFX12-NEXT: global_inv scope:SCOPE_SE7529; GFX12-NEXT: s_wait_loadcnt 0x07530; GFX12-NEXT: s_setpc_b64 s[30:31]7531;7532; GFX942-LABEL: local_atomic_fadd_noret_v2bf16:7533; GFX942: ; %bb.0:7534; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7535; GFX942-NEXT: ds_pk_add_bf16 v0, v17536; GFX942-NEXT: s_waitcnt lgkmcnt(0)7537; GFX942-NEXT: s_setpc_b64 s[30:31]7538;7539; GFX11-TRUE16-LABEL: local_atomic_fadd_noret_v2bf16:7540; GFX11-TRUE16: ; %bb.0:7541; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7542; GFX11-TRUE16-NEXT: ds_load_b32 v3, v07543; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v17544; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v17545; GFX11-TRUE16-NEXT: s_mov_b32 s0, 07546; GFX11-TRUE16-NEXT: .p2align 67547; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start7548; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=17549; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)7550; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v37551; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7552; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v2 :: v_dual_lshlrev_b32 v4, 16, v37553; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v4, v17554; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)7555; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 17556; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 17557; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v47558; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v47559; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v57560; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff7561; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff7562; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)7563; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo7564; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v57565; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo7566; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)7567; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h7568; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x07569; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v37570; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)7571; GFX11-TRUE16-NEXT: buffer_gl0_inv7572; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v37573; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v47574; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s07575; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7576; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s07577; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_17578; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end7579; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s07580; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]7581;7582; GFX11-FAKE16-LABEL: local_atomic_fadd_noret_v2bf16:7583; GFX11-FAKE16: ; %bb.0:7584; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7585; GFX11-FAKE16-NEXT: ds_load_b32 v3, v07586; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v17587; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v17588; GFX11-FAKE16-NEXT: s_mov_b32 s1, 07589; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x17590; GFX11-FAKE16-NEXT: .p2align 67591; GFX11-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start7592; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=17593; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)7594; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v37595; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7596; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v37597; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, v4, v27598; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)7599; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 17600; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 17601; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v47602; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v57603; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v57604; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff7605; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff7606; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v47607; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)7608; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo7609; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s07610; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)7611; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x70603027612; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x07613; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v37614; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)7615; GFX11-FAKE16-NEXT: buffer_gl0_inv7616; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v37617; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v47618; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s17619; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7620; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s17621; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB26_17622; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end7623; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x27624; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s17625; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]7626;7627; GFX10-LABEL: local_atomic_fadd_noret_v2bf16:7628; GFX10: ; %bb.0:7629; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7630; GFX10-NEXT: ds_read_b32 v3, v07631; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v17632; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v17633; GFX10-NEXT: s_mov_b32 s5, 07634; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start7635; GFX10-NEXT: ; =>This Inner Loop Header: Depth=17636; GFX10-NEXT: s_waitcnt lgkmcnt(0)7637; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v37638; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v37639; GFX10-NEXT: v_add_f32_e32 v4, v4, v27640; GFX10-NEXT: v_add_f32_e32 v5, v5, v17641; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 17642; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 17643; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v47644; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v57645; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v57646; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff7647; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff7648; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v47649; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo7650; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s47651; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x70603027652; GFX10-NEXT: s_waitcnt_vscnt null, 0x07653; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v47654; GFX10-NEXT: s_waitcnt lgkmcnt(0)7655; GFX10-NEXT: buffer_gl0_inv7656; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v37657; GFX10-NEXT: v_mov_b32_e32 v3, v47658; GFX10-NEXT: s_or_b32 s5, vcc_lo, s57659; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s57660; GFX10-NEXT: s_cbranch_execnz .LBB26_17661; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end7662; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s57663; GFX10-NEXT: s_setpc_b64 s[30:31]7664;7665; GFX90A-LABEL: local_atomic_fadd_noret_v2bf16:7666; GFX90A: ; %bb.0:7667; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7668; GFX90A-NEXT: ds_read_b32 v3, v07669; GFX90A-NEXT: s_mov_b64 s[6:7], 07670; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v17671; GFX90A-NEXT: s_movk_i32 s8, 0x7fff7672; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v17673; GFX90A-NEXT: s_mov_b32 s9, 0x70603027674; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start7675; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=17676; GFX90A-NEXT: s_waitcnt lgkmcnt(0)7677; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v37678; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v37679; GFX90A-NEXT: v_add_f32_e32 v4, v4, v27680; GFX90A-NEXT: v_add_f32_e32 v5, v5, v17681; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 17682; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 17683; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v47684; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v57685; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s87686; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s87687; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v57688; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v47689; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]7690; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc7691; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s97692; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v47693; GFX90A-NEXT: s_waitcnt lgkmcnt(0)7694; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v37695; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7696; GFX90A-NEXT: v_mov_b32_e32 v3, v47697; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]7698; GFX90A-NEXT: s_cbranch_execnz .LBB26_17699; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end7700; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]7701; GFX90A-NEXT: s_setpc_b64 s[30:31]7702;7703; GFX908-LABEL: local_atomic_fadd_noret_v2bf16:7704; GFX908: ; %bb.0:7705; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7706; GFX908-NEXT: ds_read_b32 v3, v07707; GFX908-NEXT: s_mov_b64 s[6:7], 07708; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v17709; GFX908-NEXT: s_movk_i32 s8, 0x7fff7710; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v17711; GFX908-NEXT: s_mov_b32 s9, 0x70603027712; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start7713; GFX908-NEXT: ; =>This Inner Loop Header: Depth=17714; GFX908-NEXT: s_waitcnt lgkmcnt(0)7715; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v37716; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v37717; GFX908-NEXT: v_add_f32_e32 v4, v4, v27718; GFX908-NEXT: v_add_f32_e32 v5, v5, v17719; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 17720; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 17721; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v47722; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v57723; GFX908-NEXT: v_add3_u32 v6, v6, v4, s87724; GFX908-NEXT: v_add3_u32 v8, v8, v5, s87725; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v57726; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v47727; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]7728; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc7729; GFX908-NEXT: v_perm_b32 v4, v5, v4, s97730; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v47731; GFX908-NEXT: s_waitcnt lgkmcnt(0)7732; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v37733; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7734; GFX908-NEXT: v_mov_b32_e32 v3, v47735; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]7736; GFX908-NEXT: s_cbranch_execnz .LBB26_17737; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end7738; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]7739; GFX908-NEXT: s_setpc_b64 s[30:31]7740;7741; GFX8-LABEL: local_atomic_fadd_noret_v2bf16:7742; GFX8: ; %bb.0:7743; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7744; GFX8-NEXT: s_mov_b32 m0, -17745; GFX8-NEXT: ds_read_b32 v3, v07746; GFX8-NEXT: s_mov_b64 s[6:7], 07747; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v17748; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v17749; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start7750; GFX8-NEXT: ; =>This Inner Loop Header: Depth=17751; GFX8-NEXT: s_waitcnt lgkmcnt(0)7752; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v37753; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v37754; GFX8-NEXT: v_add_f32_e32 v4, v4, v27755; GFX8-NEXT: v_add_f32_e32 v5, v5, v17756; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 17757; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 17758; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v47759; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v57760; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v67761; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v87762; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v57763; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v57764; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v47765; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v47766; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc7767; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]7768; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v57769; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 167770; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v47771; GFX8-NEXT: s_waitcnt lgkmcnt(0)7772; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v37773; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]7774; GFX8-NEXT: v_mov_b32_e32 v3, v47775; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]7776; GFX8-NEXT: s_cbranch_execnz .LBB26_17777; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end7778; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]7779; GFX8-NEXT: s_setpc_b64 s[30:31]7780;7781; GFX7-LABEL: local_atomic_fadd_noret_v2bf16:7782; GFX7: ; %bb.0:7783; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7784; GFX7-NEXT: s_mov_b32 m0, -17785; GFX7-NEXT: ds_read_b32 v4, v07786; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v17787; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v27788; GFX7-NEXT: s_mov_b64 s[4:5], 07789; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v17790; GFX7-NEXT: s_waitcnt lgkmcnt(0)7791; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v47792; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v47793; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v27794; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start7795; GFX7-NEXT: ; =>This Inner Loop Header: Depth=17796; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v37797; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v47798; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v37799; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v47800; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v37801; GFX7-NEXT: v_add_f32_e32 v6, v6, v27802; GFX7-NEXT: v_add_f32_e32 v5, v5, v17803; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 167804; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v67805; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 167806; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v47807; GFX7-NEXT: s_waitcnt lgkmcnt(0)7808; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v37809; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v47810; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7811; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v47812; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]7813; GFX7-NEXT: s_cbranch_execnz .LBB26_17814; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end7815; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]7816; GFX7-NEXT: s_setpc_b64 s[30:31]7817;7818; GFX6-LABEL: local_atomic_fadd_noret_v2bf16:7819; GFX6: ; %bb.0:7820; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7821; GFX6-NEXT: s_mov_b32 m0, -17822; GFX6-NEXT: ds_read_b32 v4, v07823; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v17824; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v27825; GFX6-NEXT: s_mov_b64 s[4:5], 07826; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v17827; GFX6-NEXT: s_waitcnt lgkmcnt(0)7828; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v47829; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v47830; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v27831; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start7832; GFX6-NEXT: ; =>This Inner Loop Header: Depth=17833; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v37834; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v47835; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v37836; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v47837; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v37838; GFX6-NEXT: v_add_f32_e32 v6, v6, v27839; GFX6-NEXT: v_add_f32_e32 v5, v5, v17840; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 167841; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v67842; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 167843; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v47844; GFX6-NEXT: s_waitcnt lgkmcnt(0)7845; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v37846; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v47847; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]7848; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v47849; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]7850; GFX6-NEXT: s_cbranch_execnz .LBB26_17851; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end7852; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]7853; GFX6-NEXT: s_setpc_b64 s[30:31]7854 %result = atomicrmw fadd ptr addrspace(3) %ptr, <2 x bfloat> %val seq_cst7855 ret void7856}7857 7858define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x bfloat> %val) {7859; GFX12-LABEL: local_atomic_fadd_noret_v2bf16__ofset:7860; GFX12: ; %bb.0:7861; GFX12-NEXT: s_wait_loadcnt_dscnt 0x07862; GFX12-NEXT: s_wait_expcnt 0x07863; GFX12-NEXT: s_wait_samplecnt 0x07864; GFX12-NEXT: s_wait_bvhcnt 0x07865; GFX12-NEXT: s_wait_kmcnt 0x07866; GFX12-NEXT: s_wait_storecnt 0x07867; GFX12-NEXT: ds_pk_add_bf16 v0, v1 offset:655327868; GFX12-NEXT: s_wait_dscnt 0x07869; GFX12-NEXT: global_inv scope:SCOPE_SE7870; GFX12-NEXT: s_wait_loadcnt 0x07871; GFX12-NEXT: s_setpc_b64 s[30:31]7872;7873; GFX942-LABEL: local_atomic_fadd_noret_v2bf16__ofset:7874; GFX942: ; %bb.0:7875; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7876; GFX942-NEXT: ds_pk_add_bf16 v0, v1 offset:655327877; GFX942-NEXT: s_waitcnt lgkmcnt(0)7878; GFX942-NEXT: s_setpc_b64 s[30:31]7879;7880; GFX11-TRUE16-LABEL: local_atomic_fadd_noret_v2bf16__ofset:7881; GFX11-TRUE16: ; %bb.0:7882; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7883; GFX11-TRUE16-NEXT: ds_load_b32 v3, v0 offset:655327884; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v17885; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v17886; GFX11-TRUE16-NEXT: s_mov_b32 s0, 07887; GFX11-TRUE16-NEXT: .p2align 67888; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start7889; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=17890; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)7891; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v37892; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7893; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v2 :: v_dual_lshlrev_b32 v4, 16, v37894; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v4, v17895; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)7896; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 17897; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 17898; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v47899; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v47900; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v57901; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff7902; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff7903; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)7904; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo7905; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v57906; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo7907; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)7908; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h7909; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x07910; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:655327911; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)7912; GFX11-TRUE16-NEXT: buffer_gl0_inv7913; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v37914; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v47915; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s07916; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7917; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s07918; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_17919; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end7920; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s07921; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]7922;7923; GFX11-FAKE16-LABEL: local_atomic_fadd_noret_v2bf16__ofset:7924; GFX11-FAKE16: ; %bb.0:7925; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7926; GFX11-FAKE16-NEXT: ds_load_b32 v3, v0 offset:655327927; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v17928; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v17929; GFX11-FAKE16-NEXT: s_mov_b32 s1, 07930; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x17931; GFX11-FAKE16-NEXT: .p2align 67932; GFX11-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start7933; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=17934; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)7935; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v37936; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)7937; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v37938; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, v4, v27939; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)7940; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 17941; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 17942; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v47943; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v57944; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v57945; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff7946; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff7947; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v47948; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)7949; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo7950; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s07951; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)7952; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x70603027953; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x07954; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:655327955; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)7956; GFX11-FAKE16-NEXT: buffer_gl0_inv7957; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v37958; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v47959; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s17960; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)7961; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s17962; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB27_17963; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end7964; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x27965; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s17966; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]7967;7968; GFX10-LABEL: local_atomic_fadd_noret_v2bf16__ofset:7969; GFX10: ; %bb.0:7970; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)7971; GFX10-NEXT: ds_read_b32 v3, v0 offset:655327972; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v17973; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v17974; GFX10-NEXT: s_mov_b32 s5, 07975; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start7976; GFX10-NEXT: ; =>This Inner Loop Header: Depth=17977; GFX10-NEXT: s_waitcnt lgkmcnt(0)7978; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v37979; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v37980; GFX10-NEXT: v_add_f32_e32 v4, v4, v27981; GFX10-NEXT: v_add_f32_e32 v5, v5, v17982; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 17983; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 17984; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v47985; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v57986; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v57987; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff7988; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff7989; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v47990; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo7991; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s47992; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x70603027993; GFX10-NEXT: s_waitcnt_vscnt null, 0x07994; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:655327995; GFX10-NEXT: s_waitcnt lgkmcnt(0)7996; GFX10-NEXT: buffer_gl0_inv7997; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v37998; GFX10-NEXT: v_mov_b32_e32 v3, v47999; GFX10-NEXT: s_or_b32 s5, vcc_lo, s58000; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s58001; GFX10-NEXT: s_cbranch_execnz .LBB27_18002; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end8003; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s58004; GFX10-NEXT: s_setpc_b64 s[30:31]8005;8006; GFX90A-LABEL: local_atomic_fadd_noret_v2bf16__ofset:8007; GFX90A: ; %bb.0:8008; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8009; GFX90A-NEXT: ds_read_b32 v3, v0 offset:655328010; GFX90A-NEXT: s_mov_b64 s[6:7], 08011; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v18012; GFX90A-NEXT: s_movk_i32 s8, 0x7fff8013; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v18014; GFX90A-NEXT: s_mov_b32 s9, 0x70603028015; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start8016; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=18017; GFX90A-NEXT: s_waitcnt lgkmcnt(0)8018; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v38019; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v38020; GFX90A-NEXT: v_add_f32_e32 v4, v4, v28021; GFX90A-NEXT: v_add_f32_e32 v5, v5, v18022; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 18023; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 18024; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v48025; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v58026; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s88027; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s88028; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v58029; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v48030; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]8031; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc8032; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s98033; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:655328034; GFX90A-NEXT: s_waitcnt lgkmcnt(0)8035; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v38036; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]8037; GFX90A-NEXT: v_mov_b32_e32 v3, v48038; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]8039; GFX90A-NEXT: s_cbranch_execnz .LBB27_18040; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end8041; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]8042; GFX90A-NEXT: s_setpc_b64 s[30:31]8043;8044; GFX908-LABEL: local_atomic_fadd_noret_v2bf16__ofset:8045; GFX908: ; %bb.0:8046; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8047; GFX908-NEXT: ds_read_b32 v3, v0 offset:655328048; GFX908-NEXT: s_mov_b64 s[6:7], 08049; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v18050; GFX908-NEXT: s_movk_i32 s8, 0x7fff8051; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v18052; GFX908-NEXT: s_mov_b32 s9, 0x70603028053; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start8054; GFX908-NEXT: ; =>This Inner Loop Header: Depth=18055; GFX908-NEXT: s_waitcnt lgkmcnt(0)8056; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v38057; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v38058; GFX908-NEXT: v_add_f32_e32 v4, v4, v28059; GFX908-NEXT: v_add_f32_e32 v5, v5, v18060; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 18061; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 18062; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v48063; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v58064; GFX908-NEXT: v_add3_u32 v6, v6, v4, s88065; GFX908-NEXT: v_add3_u32 v8, v8, v5, s88066; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v58067; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v48068; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]8069; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc8070; GFX908-NEXT: v_perm_b32 v4, v5, v4, s98071; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:655328072; GFX908-NEXT: s_waitcnt lgkmcnt(0)8073; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v38074; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]8075; GFX908-NEXT: v_mov_b32_e32 v3, v48076; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]8077; GFX908-NEXT: s_cbranch_execnz .LBB27_18078; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end8079; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]8080; GFX908-NEXT: s_setpc_b64 s[30:31]8081;8082; GFX8-LABEL: local_atomic_fadd_noret_v2bf16__ofset:8083; GFX8: ; %bb.0:8084; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8085; GFX8-NEXT: s_mov_b32 m0, -18086; GFX8-NEXT: ds_read_b32 v3, v0 offset:655328087; GFX8-NEXT: s_mov_b64 s[6:7], 08088; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v18089; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v18090; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start8091; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18092; GFX8-NEXT: s_waitcnt lgkmcnt(0)8093; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v38094; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v38095; GFX8-NEXT: v_add_f32_e32 v4, v4, v28096; GFX8-NEXT: v_add_f32_e32 v5, v5, v18097; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 18098; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 18099; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v48100; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v58101; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v68102; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v88103; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v58104; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v58105; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v48106; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v48107; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc8108; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]8109; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v58110; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 168111; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:655328112; GFX8-NEXT: s_waitcnt lgkmcnt(0)8113; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v38114; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]8115; GFX8-NEXT: v_mov_b32_e32 v3, v48116; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]8117; GFX8-NEXT: s_cbranch_execnz .LBB27_18118; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end8119; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]8120; GFX8-NEXT: s_setpc_b64 s[30:31]8121;8122; GFX7-LABEL: local_atomic_fadd_noret_v2bf16__ofset:8123; GFX7: ; %bb.0:8124; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8125; GFX7-NEXT: s_mov_b32 m0, -18126; GFX7-NEXT: ds_read_b32 v4, v0 offset:655328127; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v18128; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v28129; GFX7-NEXT: s_mov_b64 s[4:5], 08130; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v18131; GFX7-NEXT: s_waitcnt lgkmcnt(0)8132; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v48133; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v48134; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v28135; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start8136; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18137; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v38138; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v48139; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v38140; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v48141; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v38142; GFX7-NEXT: v_add_f32_e32 v6, v6, v28143; GFX7-NEXT: v_add_f32_e32 v5, v5, v18144; GFX7-NEXT: v_alignbit_b32 v3, v3, v4, 168145; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v68146; GFX7-NEXT: v_alignbit_b32 v4, v4, v5, 168147; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:655328148; GFX7-NEXT: s_waitcnt lgkmcnt(0)8149; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v38150; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v48151; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8152; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v48153; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]8154; GFX7-NEXT: s_cbranch_execnz .LBB27_18155; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end8156; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]8157; GFX7-NEXT: s_setpc_b64 s[30:31]8158;8159; GFX6-LABEL: local_atomic_fadd_noret_v2bf16__ofset:8160; GFX6: ; %bb.0:8161; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)8162; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v08163; GFX6-NEXT: s_mov_b32 m0, -18164; GFX6-NEXT: ds_read_b32 v4, v08165; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v18166; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v28167; GFX6-NEXT: s_mov_b64 s[4:5], 08168; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v18169; GFX6-NEXT: s_waitcnt lgkmcnt(0)8170; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v48171; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v48172; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v28173; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start8174; GFX6-NEXT: ; =>This Inner Loop Header: Depth=18175; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v38176; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v48177; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v38178; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v48179; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v38180; GFX6-NEXT: v_add_f32_e32 v6, v6, v28181; GFX6-NEXT: v_add_f32_e32 v5, v5, v18182; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 168183; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v68184; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 168185; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v48186; GFX6-NEXT: s_waitcnt lgkmcnt(0)8187; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v38188; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v48189; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]8190; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v48191; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]8192; GFX6-NEXT: s_cbranch_execnz .LBB27_18193; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end8194; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]8195; GFX6-NEXT: s_setpc_b64 s[30:31]8196 %gep = getelementptr <2 x bfloat>, ptr addrspace(3) %ptr, i32 163838197 %result = atomicrmw fadd ptr addrspace(3) %gep, <2 x bfloat> %val seq_cst8198 ret void8199}8200 8201; --------------------------------------------------------------------8202; misc8203; --------------------------------------------------------------------8204 8205define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %ptrf, i32 %idx) {8206; GFX12-LABEL: local_ds_fadd:8207; GFX12: ; %bb.0:8208; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x88209; GFX12-NEXT: s_mov_b32 s6, exec_lo8210; GFX12-NEXT: ; implicit-def: $vgpr18211; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)8212; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 08213; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v08214; GFX12-NEXT: s_wait_kmcnt 0x08215; GFX12-NEXT: s_add_co_i32 s1, s3, 48216; GFX12-NEXT: s_and_saveexec_b32 s0, vcc_lo8217; GFX12-NEXT: s_cbranch_execz .LBB28_28218; GFX12-NEXT: ; %bb.1:8219; GFX12-NEXT: s_bcnt1_i32_b32 s3, s68220; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)8221; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s38222; GFX12-NEXT: s_lshl_b32 s3, s1, 38223; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)8224; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v18225; GFX12-NEXT: ds_add_rtn_f32 v1, v2, v18226; GFX12-NEXT: s_wait_dscnt 0x08227; GFX12-NEXT: global_inv scope:SCOPE_SE8228; GFX12-NEXT: .LBB28_2:8229; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s08230; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)8231; GFX12-NEXT: s_mov_b32 s7, exec_lo8232; GFX12-NEXT: v_readfirstlane_b32 s3, v18233; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)8234; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 08235; GFX12-NEXT: s_mov_b32 s6, exec_lo8236; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v28237; GFX12-NEXT: s_cbranch_execz .LBB28_48238; GFX12-NEXT: ; %bb.3:8239; GFX12-NEXT: s_bcnt1_i32_b32 s0, s78240; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)8241; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s08242; GFX12-NEXT: s_lshl_b32 s0, s1, 48243; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)8244; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v18245; GFX12-NEXT: s_wait_loadcnt 0x08246; GFX12-NEXT: ds_add_f32 v2, v18247; GFX12-NEXT: s_wait_dscnt 0x08248; GFX12-NEXT: global_inv scope:SCOPE_SE8249; GFX12-NEXT: .LBB28_4:8250; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)8251; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s68252; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v0, v08253; GFX12-NEXT: s_mov_b32 s1, exec_lo8254; GFX12-NEXT: s_brev_b32 s0, 18255; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8256; GFX12-NEXT: v_mul_f32_e32 v0, 0x42280000, v08257; GFX12-NEXT: v_add_f32_e32 v0, s3, v08258; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)8259; GFX12-NEXT: v_cndmask_b32_e64 v1, v0, s3, vcc_lo8260; GFX12-NEXT: ; implicit-def: $vgpr08261; GFX12-NEXT: .LBB28_5: ; %ComputeLoop8262; GFX12-NEXT: ; =>This Inner Loop Header: Depth=18263; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)8264; GFX12-NEXT: s_ctz_i32_b32 s3, s18265; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)8266; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)8267; GFX12-NEXT: v_readlane_b32 s6, v1, s38268; GFX12-NEXT: v_writelane_b32 v0, s0, s38269; GFX12-NEXT: s_lshl_b32 s3, 1, s38270; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)8271; GFX12-NEXT: s_and_not1_b32 s1, s1, s38272; GFX12-NEXT: s_add_f32 s0, s0, s68273; GFX12-NEXT: s_cbranch_scc1 .LBB28_58274; GFX12-NEXT: ; %bb.6: ; %ComputeEnd8275; GFX12-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 08276; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)8277; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v18278; GFX12-NEXT: ; implicit-def: $vgpr18279; GFX12-NEXT: s_and_saveexec_b32 s1, vcc_lo8280; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)8281; GFX12-NEXT: s_xor_b32 s1, exec_lo, s18282; GFX12-NEXT: s_cbranch_execz .LBB28_88283; GFX12-NEXT: ; %bb.7:8284; GFX12-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s08285; GFX12-NEXT: s_wait_loadcnt 0x08286; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v28287; GFX12-NEXT: s_wait_dscnt 0x08288; GFX12-NEXT: global_inv scope:SCOPE_SE8289; GFX12-NEXT: .LBB28_8:8290; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)8291; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s18292; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x08293; GFX12-NEXT: v_readfirstlane_b32 s2, v18294; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)8295; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8296; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v08297; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo8298; GFX12-NEXT: s_wait_kmcnt 0x08299; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]8300; GFX12-NEXT: s_endpgm8301;8302; GFX942-LABEL: local_ds_fadd:8303; GFX942: ; %bb.0:8304; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x88305; GFX942-NEXT: s_mov_b64 s[0:1], exec8306; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 08307; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v08308; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08309; GFX942-NEXT: s_waitcnt lgkmcnt(0)8310; GFX942-NEXT: s_add_i32 s3, s3, 48311; GFX942-NEXT: ; implicit-def: $vgpr18312; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc8313; GFX942-NEXT: s_cbranch_execz .LBB28_28314; GFX942-NEXT: ; %bb.1:8315; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]8316; GFX942-NEXT: s_lshl_b32 s8, s3, 38317; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s08318; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v18319; GFX942-NEXT: v_mov_b32_e32 v2, s88320; GFX942-NEXT: ds_add_rtn_f32 v1, v2, v18321; GFX942-NEXT: s_waitcnt lgkmcnt(0)8322; GFX942-NEXT: .LBB28_2:8323; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]8324; GFX942-NEXT: s_mov_b64 s[8:9], exec8325; GFX942-NEXT: v_readfirstlane_b32 s10, v18326; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 08327; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v18328; GFX942-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v18329; GFX942-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]8330; GFX942-NEXT: s_cbranch_execz .LBB28_48331; GFX942-NEXT: ; %bb.3:8332; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[8:9]8333; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s08334; GFX942-NEXT: s_lshl_b32 s0, s3, 48335; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v18336; GFX942-NEXT: v_mov_b32_e32 v2, s08337; GFX942-NEXT: ds_add_f32 v2, v18338; GFX942-NEXT: s_waitcnt lgkmcnt(0)8339; GFX942-NEXT: .LBB28_4:8340; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]8341; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v0, v08342; GFX942-NEXT: v_mul_f32_e32 v0, 0x42280000, v08343; GFX942-NEXT: v_add_f32_e32 v0, s10, v08344; GFX942-NEXT: v_mov_b32_e32 v1, s108345; GFX942-NEXT: s_mov_b64 s[0:1], exec8346; GFX942-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc8347; GFX942-NEXT: v_bfrev_b32_e32 v1, 18348; GFX942-NEXT: ; implicit-def: $vgpr08349; GFX942-NEXT: .LBB28_5: ; %ComputeLoop8350; GFX942-NEXT: ; =>This Inner Loop Header: Depth=18351; GFX942-NEXT: s_ff1_i32_b64 s3, s[0:1]8352; GFX942-NEXT: v_readfirstlane_b32 s6, v18353; GFX942-NEXT: s_mov_b32 m0, s38354; GFX942-NEXT: v_readlane_b32 s8, v2, s38355; GFX942-NEXT: v_writelane_b32 v0, s6, m08356; GFX942-NEXT: s_lshl_b64 s[6:7], 1, s38357; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]8358; GFX942-NEXT: v_add_f32_e32 v1, s8, v18359; GFX942-NEXT: s_cbranch_scc1 .LBB28_58360; GFX942-NEXT: ; %bb.6: ; %ComputeEnd8361; GFX942-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 08362; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v28363; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v28364; GFX942-NEXT: ; implicit-def: $vgpr28365; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc8366; GFX942-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8367; GFX942-NEXT: s_cbranch_execz .LBB28_88368; GFX942-NEXT: ; %bb.7:8369; GFX942-NEXT: v_mov_b32_e32 v2, s28370; GFX942-NEXT: ds_add_rtn_f32 v2, v2, v18371; GFX942-NEXT: s_waitcnt lgkmcnt(0)8372; GFX942-NEXT: .LBB28_8:8373; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]8374; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x08375; GFX942-NEXT: v_readfirstlane_b32 s2, v28376; GFX942-NEXT: v_mov_b32_e32 v1, 08377; GFX942-NEXT: s_nop 08378; GFX942-NEXT: v_add_f32_e32 v0, s2, v08379; GFX942-NEXT: v_mov_b32_e32 v2, s28380; GFX942-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc8381; GFX942-NEXT: s_waitcnt lgkmcnt(0)8382; GFX942-NEXT: global_store_dword v1, v0, s[0:1]8383; GFX942-NEXT: s_endpgm8384;8385; GFX11-LABEL: local_ds_fadd:8386; GFX11: ; %bb.0:8387; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x88388; GFX11-NEXT: s_mov_b32 s6, exec_lo8389; GFX11-NEXT: ; implicit-def: $vgpr18390; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)8391; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 08392; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v08393; GFX11-NEXT: s_waitcnt lgkmcnt(0)8394; GFX11-NEXT: s_add_i32 s1, s3, 48395; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo8396; GFX11-NEXT: s_cbranch_execz .LBB28_28397; GFX11-NEXT: ; %bb.1:8398; GFX11-NEXT: s_bcnt1_i32_b32 s3, s68399; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8400; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s38401; GFX11-NEXT: s_lshl_b32 s3, s1, 38402; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)8403; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v18404; GFX11-NEXT: ds_add_rtn_f32 v1, v2, v18405; GFX11-NEXT: s_waitcnt lgkmcnt(0)8406; GFX11-NEXT: buffer_gl0_inv8407; GFX11-NEXT: .LBB28_2:8408; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s08409; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)8410; GFX11-NEXT: s_mov_b32 s7, exec_lo8411; GFX11-NEXT: v_readfirstlane_b32 s3, v18412; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 08413; GFX11-NEXT: s_mov_b32 s6, exec_lo8414; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v28415; GFX11-NEXT: s_cbranch_execz .LBB28_48416; GFX11-NEXT: ; %bb.3:8417; GFX11-NEXT: s_bcnt1_i32_b32 s0, s78418; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)8419; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s08420; GFX11-NEXT: s_lshl_b32 s0, s1, 48421; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)8422; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v18423; GFX11-NEXT: ds_add_f32 v2, v18424; GFX11-NEXT: s_waitcnt lgkmcnt(0)8425; GFX11-NEXT: buffer_gl0_inv8426; GFX11-NEXT: .LBB28_4:8427; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s68428; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v08429; GFX11-NEXT: v_bfrev_b32_e32 v1, 18430; GFX11-NEXT: s_mov_b32 s0, exec_lo8431; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)8432; GFX11-NEXT: v_mul_f32_e32 v0, 0x42280000, v08433; GFX11-NEXT: v_add_f32_e32 v0, s3, v08434; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)8435; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, s3, vcc_lo8436; GFX11-NEXT: ; implicit-def: $vgpr08437; GFX11-NEXT: .LBB28_5: ; %ComputeLoop8438; GFX11-NEXT: ; =>This Inner Loop Header: Depth=18439; GFX11-NEXT: s_ctz_i32_b32 s1, s08440; GFX11-NEXT: v_readfirstlane_b32 s3, v18441; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)8442; GFX11-NEXT: v_readlane_b32 s6, v2, s18443; GFX11-NEXT: v_writelane_b32 v0, s3, s18444; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)8445; GFX11-NEXT: v_add_f32_e32 v1, s6, v18446; GFX11-NEXT: s_lshl_b32 s1, 1, s18447; GFX11-NEXT: s_and_not1_b32 s0, s0, s18448; GFX11-NEXT: s_cbranch_scc1 .LBB28_58449; GFX11-NEXT: ; %bb.6: ; %ComputeEnd8450; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 08451; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)8452; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v28453; GFX11-NEXT: ; implicit-def: $vgpr28454; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo8455; GFX11-NEXT: s_xor_b32 s0, exec_lo, s08456; GFX11-NEXT: s_cbranch_execz .LBB28_88457; GFX11-NEXT: ; %bb.7:8458; GFX11-NEXT: v_mov_b32_e32 v2, s28459; GFX11-NEXT: ds_add_rtn_f32 v2, v2, v18460; GFX11-NEXT: s_waitcnt lgkmcnt(0)8461; GFX11-NEXT: buffer_gl0_inv8462; GFX11-NEXT: .LBB28_8:8463; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s08464; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x08465; GFX11-NEXT: v_readfirstlane_b32 s2, v28466; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)8467; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v08468; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo8469; GFX11-NEXT: s_waitcnt lgkmcnt(0)8470; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]8471; GFX11-NEXT: s_endpgm8472;8473; GFX10-LABEL: local_ds_fadd:8474; GFX10: ; %bb.0:8475; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x88476; GFX10-NEXT: s_mov_b32 s6, exec_lo8477; GFX10-NEXT: ; implicit-def: $vgpr18478; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 08479; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v08480; GFX10-NEXT: s_waitcnt lgkmcnt(0)8481; GFX10-NEXT: s_add_i32 s1, s3, 48482; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo8483; GFX10-NEXT: s_cbranch_execz .LBB28_28484; GFX10-NEXT: ; %bb.1:8485; GFX10-NEXT: s_bcnt1_i32_b32 s3, s68486; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s38487; GFX10-NEXT: s_lshl_b32 s3, s1, 38488; GFX10-NEXT: v_mov_b32_e32 v2, s38489; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v18490; GFX10-NEXT: ds_add_rtn_f32 v1, v2, v18491; GFX10-NEXT: s_waitcnt lgkmcnt(0)8492; GFX10-NEXT: buffer_gl0_inv8493; GFX10-NEXT: .LBB28_2:8494; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)8495; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s08496; GFX10-NEXT: s_mov_b32 s7, exec_lo8497; GFX10-NEXT: v_readfirstlane_b32 s3, v18498; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 08499; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v28500; GFX10-NEXT: s_and_saveexec_b32 s6, s08501; GFX10-NEXT: s_cbranch_execz .LBB28_48502; GFX10-NEXT: ; %bb.3:8503; GFX10-NEXT: s_bcnt1_i32_b32 s0, s78504; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s08505; GFX10-NEXT: s_lshl_b32 s0, s1, 48506; GFX10-NEXT: v_mov_b32_e32 v2, s08507; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v18508; GFX10-NEXT: s_waitcnt_vscnt null, 0x08509; GFX10-NEXT: ds_add_f32 v2, v18510; GFX10-NEXT: s_waitcnt lgkmcnt(0)8511; GFX10-NEXT: buffer_gl0_inv8512; GFX10-NEXT: .LBB28_4:8513; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)8514; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s68515; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v08516; GFX10-NEXT: v_bfrev_b32_e32 v1, 18517; GFX10-NEXT: s_mov_b32 s0, exec_lo8518; GFX10-NEXT: v_mul_f32_e32 v0, 0x42280000, v08519; GFX10-NEXT: v_add_f32_e32 v0, s3, v08520; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, s3, vcc_lo8521; GFX10-NEXT: ; implicit-def: $vgpr08522; GFX10-NEXT: .LBB28_5: ; %ComputeLoop8523; GFX10-NEXT: ; =>This Inner Loop Header: Depth=18524; GFX10-NEXT: s_ff1_i32_b32 s1, s08525; GFX10-NEXT: v_readfirstlane_b32 s3, v18526; GFX10-NEXT: v_readlane_b32 s6, v2, s18527; GFX10-NEXT: v_writelane_b32 v0, s3, s18528; GFX10-NEXT: v_add_f32_e32 v1, s6, v18529; GFX10-NEXT: s_lshl_b32 s1, 1, s18530; GFX10-NEXT: s_andn2_b32 s0, s0, s18531; GFX10-NEXT: s_cbranch_scc1 .LBB28_58532; GFX10-NEXT: ; %bb.6: ; %ComputeEnd8533; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 08534; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v28535; GFX10-NEXT: ; implicit-def: $vgpr28536; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo8537; GFX10-NEXT: s_xor_b32 s0, exec_lo, s08538; GFX10-NEXT: s_cbranch_execz .LBB28_88539; GFX10-NEXT: ; %bb.7:8540; GFX10-NEXT: v_mov_b32_e32 v2, s28541; GFX10-NEXT: s_waitcnt_vscnt null, 0x08542; GFX10-NEXT: ds_add_rtn_f32 v2, v2, v18543; GFX10-NEXT: s_waitcnt lgkmcnt(0)8544; GFX10-NEXT: buffer_gl0_inv8545; GFX10-NEXT: .LBB28_8:8546; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)8547; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s08548; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x08549; GFX10-NEXT: v_readfirstlane_b32 s2, v28550; GFX10-NEXT: v_mov_b32_e32 v1, 08551; GFX10-NEXT: v_add_f32_e32 v0, s2, v08552; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo8553; GFX10-NEXT: s_waitcnt lgkmcnt(0)8554; GFX10-NEXT: global_store_dword v1, v0, s[0:1]8555; GFX10-NEXT: s_endpgm8556;8557; GFX90A-LABEL: local_ds_fadd:8558; GFX90A: ; %bb.0:8559; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x88560; GFX90A-NEXT: s_mov_b64 s[0:1], exec8561; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 08562; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v08563; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08564; GFX90A-NEXT: s_waitcnt lgkmcnt(0)8565; GFX90A-NEXT: s_add_i32 s3, s3, 48566; GFX90A-NEXT: ; implicit-def: $vgpr18567; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc8568; GFX90A-NEXT: s_cbranch_execz .LBB28_28569; GFX90A-NEXT: ; %bb.1:8570; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]8571; GFX90A-NEXT: s_lshl_b32 s8, s3, 38572; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s08573; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v18574; GFX90A-NEXT: v_mov_b32_e32 v2, s88575; GFX90A-NEXT: ds_add_rtn_f32 v1, v2, v18576; GFX90A-NEXT: s_waitcnt lgkmcnt(0)8577; GFX90A-NEXT: .LBB28_2:8578; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]8579; GFX90A-NEXT: s_mov_b64 s[8:9], exec8580; GFX90A-NEXT: v_readfirstlane_b32 s10, v18581; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 08582; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v18583; GFX90A-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v18584; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]8585; GFX90A-NEXT: s_cbranch_execz .LBB28_48586; GFX90A-NEXT: ; %bb.3:8587; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[8:9]8588; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s08589; GFX90A-NEXT: s_lshl_b32 s0, s3, 48590; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v18591; GFX90A-NEXT: v_mov_b32_e32 v2, s08592; GFX90A-NEXT: ds_add_f32 v2, v18593; GFX90A-NEXT: s_waitcnt lgkmcnt(0)8594; GFX90A-NEXT: .LBB28_4:8595; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]8596; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v08597; GFX90A-NEXT: v_mul_f32_e32 v0, 0x42280000, v08598; GFX90A-NEXT: v_add_f32_e32 v0, s10, v08599; GFX90A-NEXT: v_mov_b32_e32 v1, s108600; GFX90A-NEXT: s_mov_b64 s[0:1], exec8601; GFX90A-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc8602; GFX90A-NEXT: v_bfrev_b32_e32 v1, 18603; GFX90A-NEXT: ; implicit-def: $vgpr08604; GFX90A-NEXT: .LBB28_5: ; %ComputeLoop8605; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=18606; GFX90A-NEXT: s_ff1_i32_b64 s3, s[0:1]8607; GFX90A-NEXT: v_readfirstlane_b32 s6, v18608; GFX90A-NEXT: s_mov_b32 m0, s38609; GFX90A-NEXT: v_readlane_b32 s8, v2, s38610; GFX90A-NEXT: v_writelane_b32 v0, s6, m08611; GFX90A-NEXT: s_lshl_b64 s[6:7], 1, s38612; GFX90A-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]8613; GFX90A-NEXT: v_add_f32_e32 v1, s8, v18614; GFX90A-NEXT: s_cbranch_scc1 .LBB28_58615; GFX90A-NEXT: ; %bb.6: ; %ComputeEnd8616; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 08617; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v28618; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v28619; GFX90A-NEXT: ; implicit-def: $vgpr28620; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc8621; GFX90A-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8622; GFX90A-NEXT: s_cbranch_execz .LBB28_88623; GFX90A-NEXT: ; %bb.7:8624; GFX90A-NEXT: v_mov_b32_e32 v2, s28625; GFX90A-NEXT: ds_add_rtn_f32 v2, v2, v18626; GFX90A-NEXT: s_waitcnt lgkmcnt(0)8627; GFX90A-NEXT: .LBB28_8:8628; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]8629; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x08630; GFX90A-NEXT: v_readfirstlane_b32 s2, v28631; GFX90A-NEXT: v_add_f32_e32 v0, s2, v08632; GFX90A-NEXT: v_mov_b32_e32 v2, s28633; GFX90A-NEXT: v_mov_b32_e32 v1, 08634; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc8635; GFX90A-NEXT: s_waitcnt lgkmcnt(0)8636; GFX90A-NEXT: global_store_dword v1, v0, s[0:1]8637; GFX90A-NEXT: s_endpgm8638;8639; GFX908-LABEL: local_ds_fadd:8640; GFX908: ; %bb.0:8641; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x88642; GFX908-NEXT: s_mov_b64 s[0:1], exec8643; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 08644; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v08645; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08646; GFX908-NEXT: s_waitcnt lgkmcnt(0)8647; GFX908-NEXT: s_add_i32 s3, s3, 48648; GFX908-NEXT: ; implicit-def: $vgpr18649; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc8650; GFX908-NEXT: s_cbranch_execz .LBB28_28651; GFX908-NEXT: ; %bb.1:8652; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]8653; GFX908-NEXT: s_lshl_b32 s8, s3, 38654; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s08655; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v18656; GFX908-NEXT: v_mov_b32_e32 v2, s88657; GFX908-NEXT: ds_add_rtn_f32 v1, v2, v18658; GFX908-NEXT: s_waitcnt lgkmcnt(0)8659; GFX908-NEXT: .LBB28_2:8660; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]8661; GFX908-NEXT: s_mov_b64 s[8:9], exec8662; GFX908-NEXT: v_readfirstlane_b32 s10, v18663; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 08664; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v18665; GFX908-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v18666; GFX908-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]8667; GFX908-NEXT: s_cbranch_execz .LBB28_48668; GFX908-NEXT: ; %bb.3:8669; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[8:9]8670; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s08671; GFX908-NEXT: s_lshl_b32 s0, s3, 48672; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v18673; GFX908-NEXT: v_mov_b32_e32 v2, s08674; GFX908-NEXT: ds_add_f32 v2, v18675; GFX908-NEXT: s_waitcnt lgkmcnt(0)8676; GFX908-NEXT: .LBB28_4:8677; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]8678; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v08679; GFX908-NEXT: v_mul_f32_e32 v0, 0x42280000, v08680; GFX908-NEXT: v_add_f32_e32 v0, s10, v08681; GFX908-NEXT: v_mov_b32_e32 v1, s108682; GFX908-NEXT: s_mov_b64 s[0:1], exec8683; GFX908-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc8684; GFX908-NEXT: v_bfrev_b32_e32 v1, 18685; GFX908-NEXT: ; implicit-def: $vgpr08686; GFX908-NEXT: .LBB28_5: ; %ComputeLoop8687; GFX908-NEXT: ; =>This Inner Loop Header: Depth=18688; GFX908-NEXT: s_ff1_i32_b64 s3, s[0:1]8689; GFX908-NEXT: v_readfirstlane_b32 s6, v18690; GFX908-NEXT: s_mov_b32 m0, s38691; GFX908-NEXT: v_readlane_b32 s8, v2, s38692; GFX908-NEXT: v_writelane_b32 v0, s6, m08693; GFX908-NEXT: s_lshl_b64 s[6:7], 1, s38694; GFX908-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]8695; GFX908-NEXT: v_add_f32_e32 v1, s8, v18696; GFX908-NEXT: s_cbranch_scc1 .LBB28_58697; GFX908-NEXT: ; %bb.6: ; %ComputeEnd8698; GFX908-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 08699; GFX908-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v28700; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v28701; GFX908-NEXT: ; implicit-def: $vgpr28702; GFX908-NEXT: s_and_saveexec_b64 s[0:1], vcc8703; GFX908-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8704; GFX908-NEXT: s_cbranch_execz .LBB28_88705; GFX908-NEXT: ; %bb.7:8706; GFX908-NEXT: v_mov_b32_e32 v2, s28707; GFX908-NEXT: ds_add_rtn_f32 v2, v2, v18708; GFX908-NEXT: s_waitcnt lgkmcnt(0)8709; GFX908-NEXT: .LBB28_8:8710; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]8711; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x08712; GFX908-NEXT: v_readfirstlane_b32 s2, v28713; GFX908-NEXT: v_add_f32_e32 v0, s2, v08714; GFX908-NEXT: v_mov_b32_e32 v2, s28715; GFX908-NEXT: v_mov_b32_e32 v1, 08716; GFX908-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc8717; GFX908-NEXT: s_waitcnt lgkmcnt(0)8718; GFX908-NEXT: global_store_dword v1, v0, s[0:1]8719; GFX908-NEXT: s_endpgm8720;8721; GFX8-LABEL: local_ds_fadd:8722; GFX8: ; %bb.0:8723; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x88724; GFX8-NEXT: s_mov_b64 s[0:1], exec8725; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 08726; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v08727; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08728; GFX8-NEXT: s_waitcnt lgkmcnt(0)8729; GFX8-NEXT: s_add_i32 s3, s3, 48730; GFX8-NEXT: ; implicit-def: $vgpr18731; GFX8-NEXT: s_mov_b32 m0, -18732; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc8733; GFX8-NEXT: s_cbranch_execz .LBB28_28734; GFX8-NEXT: ; %bb.1:8735; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]8736; GFX8-NEXT: s_lshl_b32 s8, s3, 38737; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s08738; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v18739; GFX8-NEXT: v_mov_b32_e32 v2, s88740; GFX8-NEXT: ds_add_rtn_f32 v1, v2, v18741; GFX8-NEXT: s_waitcnt lgkmcnt(0)8742; GFX8-NEXT: .LBB28_2:8743; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]8744; GFX8-NEXT: s_mov_b64 s[8:9], exec8745; GFX8-NEXT: v_readfirstlane_b32 s10, v18746; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 08747; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v18748; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v18749; GFX8-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]8750; GFX8-NEXT: s_cbranch_execz .LBB28_48751; GFX8-NEXT: ; %bb.3:8752; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[8:9]8753; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s08754; GFX8-NEXT: s_lshl_b32 s0, s3, 48755; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v18756; GFX8-NEXT: v_mov_b32_e32 v2, s08757; GFX8-NEXT: ds_add_f32 v2, v18758; GFX8-NEXT: s_waitcnt lgkmcnt(0)8759; GFX8-NEXT: .LBB28_4:8760; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]8761; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v08762; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v08763; GFX8-NEXT: v_add_f32_e32 v0, s10, v08764; GFX8-NEXT: v_mov_b32_e32 v1, s108765; GFX8-NEXT: s_mov_b64 s[0:1], exec8766; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc8767; GFX8-NEXT: v_bfrev_b32_e32 v1, 18768; GFX8-NEXT: ; implicit-def: $vgpr08769; GFX8-NEXT: .LBB28_5: ; %ComputeLoop8770; GFX8-NEXT: ; =>This Inner Loop Header: Depth=18771; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]8772; GFX8-NEXT: v_readfirstlane_b32 s6, v18773; GFX8-NEXT: s_mov_b32 m0, s38774; GFX8-NEXT: v_readlane_b32 s8, v2, s38775; GFX8-NEXT: v_writelane_b32 v0, s6, m08776; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s38777; GFX8-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]8778; GFX8-NEXT: v_add_f32_e32 v1, s8, v18779; GFX8-NEXT: s_cbranch_scc1 .LBB28_58780; GFX8-NEXT: ; %bb.6: ; %ComputeEnd8781; GFX8-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 08782; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v28783; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v28784; GFX8-NEXT: ; implicit-def: $vgpr28785; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc8786; GFX8-NEXT: s_xor_b64 s[0:1], exec, s[0:1]8787; GFX8-NEXT: s_cbranch_execz .LBB28_88788; GFX8-NEXT: ; %bb.7:8789; GFX8-NEXT: v_mov_b32_e32 v2, s28790; GFX8-NEXT: s_mov_b32 m0, -18791; GFX8-NEXT: ds_add_rtn_f32 v2, v2, v18792; GFX8-NEXT: s_waitcnt lgkmcnt(0)8793; GFX8-NEXT: .LBB28_8:8794; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]8795; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x08796; GFX8-NEXT: v_readfirstlane_b32 s2, v28797; GFX8-NEXT: v_add_f32_e32 v0, s2, v08798; GFX8-NEXT: v_mov_b32_e32 v1, s28799; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc8800; GFX8-NEXT: s_waitcnt lgkmcnt(0)8801; GFX8-NEXT: v_mov_b32_e32 v0, s08802; GFX8-NEXT: v_mov_b32_e32 v1, s18803; GFX8-NEXT: flat_store_dword v[0:1], v28804; GFX8-NEXT: s_endpgm8805;8806; GFX7-LABEL: local_ds_fadd:8807; GFX7: ; %bb.0:8808; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x28809; GFX7-NEXT: s_mov_b64 s[0:1], exec8810; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 08811; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v08812; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08813; GFX7-NEXT: s_waitcnt lgkmcnt(0)8814; GFX7-NEXT: s_add_i32 s3, s3, 48815; GFX7-NEXT: ; implicit-def: $vgpr18816; GFX7-NEXT: s_mov_b32 m0, -18817; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc8818; GFX7-NEXT: s_cbranch_execz .LBB28_48819; GFX7-NEXT: ; %bb.1:8820; GFX7-NEXT: s_lshl_b32 s8, s3, 38821; GFX7-NEXT: v_mov_b32_e32 v2, s88822; GFX7-NEXT: ds_read_b32 v1, v28823; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]8824; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s08825; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v38826; GFX7-NEXT: s_mov_b64 s[8:9], 08827; GFX7-NEXT: .LBB28_2: ; %atomicrmw.start8828; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18829; GFX7-NEXT: s_waitcnt lgkmcnt(0)8830; GFX7-NEXT: v_mov_b32_e32 v4, v18831; GFX7-NEXT: v_add_f32_e32 v1, v4, v38832; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v18833; GFX7-NEXT: s_waitcnt lgkmcnt(0)8834; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v48835; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]8836; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]8837; GFX7-NEXT: s_cbranch_execnz .LBB28_28838; GFX7-NEXT: ; %bb.3: ; %Flow228839; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]8840; GFX7-NEXT: .LBB28_4: ; %Flow238841; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]8842; GFX7-NEXT: s_mov_b64 s[8:9], exec8843; GFX7-NEXT: v_readfirstlane_b32 s10, v18844; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 08845; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v18846; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v18847; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]8848; GFX7-NEXT: s_cbranch_execz .LBB28_78849; GFX7-NEXT: ; %bb.5:8850; GFX7-NEXT: s_lshl_b32 s0, s3, 48851; GFX7-NEXT: v_mov_b32_e32 v1, s08852; GFX7-NEXT: ds_read_b32 v3, v18853; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[8:9]8854; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s08855; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v28856; GFX7-NEXT: s_mov_b64 s[8:9], 08857; GFX7-NEXT: .LBB28_6: ; %atomicrmw.start28858; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18859; GFX7-NEXT: s_waitcnt lgkmcnt(0)8860; GFX7-NEXT: v_add_f32_e32 v4, v3, v28861; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v48862; GFX7-NEXT: s_waitcnt lgkmcnt(0)8863; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v38864; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]8865; GFX7-NEXT: v_mov_b32_e32 v3, v48866; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]8867; GFX7-NEXT: s_cbranch_execnz .LBB28_68868; GFX7-NEXT: .LBB28_7: ; %Flow218869; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]8870; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v08871; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v08872; GFX7-NEXT: v_add_f32_e32 v0, s10, v08873; GFX7-NEXT: v_mov_b32_e32 v1, s108874; GFX7-NEXT: s_mov_b64 s[0:1], exec8875; GFX7-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc8876; GFX7-NEXT: v_bfrev_b32_e32 v1, 18877; GFX7-NEXT: ; implicit-def: $vgpr08878; GFX7-NEXT: .LBB28_8: ; %ComputeLoop8879; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18880; GFX7-NEXT: s_ff1_i32_b64 s3, s[0:1]8881; GFX7-NEXT: s_lshl_b64 s[6:7], 1, s38882; GFX7-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]8883; GFX7-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 08884; GFX7-NEXT: v_readfirstlane_b32 s8, v18885; GFX7-NEXT: v_readlane_b32 s9, v2, s38886; GFX7-NEXT: s_mov_b32 m0, s38887; GFX7-NEXT: v_writelane_b32 v0, s8, m08888; GFX7-NEXT: v_add_f32_e32 v1, s9, v18889; GFX7-NEXT: s_and_b64 vcc, exec, s[6:7]8890; GFX7-NEXT: s_cbranch_vccnz .LBB28_88891; GFX7-NEXT: ; %bb.9: ; %ComputeEnd8892; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 08893; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v28894; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v28895; GFX7-NEXT: ; implicit-def: $vgpr28896; GFX7-NEXT: s_and_saveexec_b64 s[0:1], vcc8897; GFX7-NEXT: s_xor_b64 s[6:7], exec, s[0:1]8898; GFX7-NEXT: s_cbranch_execz .LBB28_138899; GFX7-NEXT: ; %bb.10:8900; GFX7-NEXT: v_mov_b32_e32 v3, s28901; GFX7-NEXT: s_mov_b32 m0, -18902; GFX7-NEXT: ds_read_b32 v2, v38903; GFX7-NEXT: s_mov_b64 s[2:3], 08904; GFX7-NEXT: .LBB28_11: ; %atomicrmw.start88905; GFX7-NEXT: ; =>This Inner Loop Header: Depth=18906; GFX7-NEXT: s_waitcnt lgkmcnt(0)8907; GFX7-NEXT: v_mov_b32_e32 v4, v28908; GFX7-NEXT: v_add_f32_e32 v2, v4, v18909; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v3, v4, v28910; GFX7-NEXT: s_waitcnt lgkmcnt(0)8911; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v48912; GFX7-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]8913; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]8914; GFX7-NEXT: s_cbranch_execnz .LBB28_118915; GFX7-NEXT: ; %bb.12: ; %Flow8916; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]8917; GFX7-NEXT: .LBB28_13: ; %Flow198918; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]8919; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x08920; GFX7-NEXT: v_readfirstlane_b32 s4, v28921; GFX7-NEXT: v_add_f32_e32 v0, s4, v08922; GFX7-NEXT: v_mov_b32_e32 v1, s48923; GFX7-NEXT: s_mov_b32 s3, 0xf0008924; GFX7-NEXT: s_mov_b32 s2, -18925; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc8926; GFX7-NEXT: s_waitcnt lgkmcnt(0)8927; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 08928; GFX7-NEXT: s_endpgm8929;8930; GFX6-LABEL: local_ds_fadd:8931; GFX6: ; %bb.0:8932; GFX6-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x28933; GFX6-NEXT: s_mov_b64 s[0:1], exec8934; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 08935; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v08936; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v08937; GFX6-NEXT: s_waitcnt lgkmcnt(0)8938; GFX6-NEXT: s_add_i32 s3, s3, 48939; GFX6-NEXT: ; implicit-def: $vgpr18940; GFX6-NEXT: s_mov_b32 m0, -18941; GFX6-NEXT: s_and_saveexec_b64 s[6:7], vcc8942; GFX6-NEXT: s_cbranch_execz .LBB28_48943; GFX6-NEXT: ; %bb.1:8944; GFX6-NEXT: s_lshl_b32 s8, s3, 38945; GFX6-NEXT: v_mov_b32_e32 v2, s88946; GFX6-NEXT: ds_read_b32 v1, v28947; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]8948; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v3, s08949; GFX6-NEXT: v_mul_f32_e32 v3, 0x42280000, v38950; GFX6-NEXT: s_mov_b64 s[8:9], 08951; GFX6-NEXT: .LBB28_2: ; %atomicrmw.start8952; GFX6-NEXT: ; =>This Inner Loop Header: Depth=18953; GFX6-NEXT: s_waitcnt lgkmcnt(0)8954; GFX6-NEXT: v_mov_b32_e32 v4, v18955; GFX6-NEXT: v_add_f32_e32 v1, v4, v38956; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v18957; GFX6-NEXT: s_waitcnt lgkmcnt(0)8958; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v48959; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]8960; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]8961; GFX6-NEXT: s_cbranch_execnz .LBB28_28962; GFX6-NEXT: ; %bb.3: ; %Flow208963; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]8964; GFX6-NEXT: .LBB28_4: ; %Flow218965; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]8966; GFX6-NEXT: s_mov_b64 s[8:9], exec8967; GFX6-NEXT: v_readfirstlane_b32 s10, v18968; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 08969; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v18970; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v18971; GFX6-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]8972; GFX6-NEXT: s_cbranch_execz .LBB28_78973; GFX6-NEXT: ; %bb.5:8974; GFX6-NEXT: s_lshl_b32 s0, s3, 48975; GFX6-NEXT: v_mov_b32_e32 v1, s08976; GFX6-NEXT: ds_read_b32 v3, v18977; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[8:9]8978; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s08979; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v28980; GFX6-NEXT: s_mov_b64 s[8:9], 08981; GFX6-NEXT: .LBB28_6: ; %atomicrmw.start28982; GFX6-NEXT: ; =>This Inner Loop Header: Depth=18983; GFX6-NEXT: s_waitcnt lgkmcnt(0)8984; GFX6-NEXT: v_add_f32_e32 v4, v3, v28985; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v48986; GFX6-NEXT: s_waitcnt lgkmcnt(0)8987; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v38988; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]8989; GFX6-NEXT: v_mov_b32_e32 v3, v48990; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]8991; GFX6-NEXT: s_cbranch_execnz .LBB28_68992; GFX6-NEXT: .LBB28_7: ; %Flow198993; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]8994; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, v08995; GFX6-NEXT: v_mul_f32_e32 v0, 0x42280000, v08996; GFX6-NEXT: v_add_f32_e32 v0, s10, v08997; GFX6-NEXT: v_mov_b32_e32 v1, s108998; GFX6-NEXT: s_mov_b64 s[0:1], exec8999; GFX6-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc9000; GFX6-NEXT: v_bfrev_b32_e32 v1, 19001; GFX6-NEXT: ; implicit-def: $vgpr09002; GFX6-NEXT: .LBB28_8: ; %ComputeLoop9003; GFX6-NEXT: ; =>This Inner Loop Header: Depth=19004; GFX6-NEXT: s_ff1_i32_b64 s3, s[0:1]9005; GFX6-NEXT: s_lshl_b64 s[6:7], 1, s39006; GFX6-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]9007; GFX6-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 09008; GFX6-NEXT: v_readfirstlane_b32 s8, v19009; GFX6-NEXT: v_readlane_b32 s9, v2, s39010; GFX6-NEXT: s_mov_b32 m0, s39011; GFX6-NEXT: v_writelane_b32 v0, s8, m09012; GFX6-NEXT: v_add_f32_e32 v1, s9, v19013; GFX6-NEXT: s_and_b64 vcc, exec, s[6:7]9014; GFX6-NEXT: s_cbranch_vccnz .LBB28_89015; GFX6-NEXT: ; %bb.9: ; %ComputeEnd9016; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 09017; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v29018; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v29019; GFX6-NEXT: ; implicit-def: $vgpr29020; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc9021; GFX6-NEXT: s_xor_b64 s[6:7], exec, s[0:1]9022; GFX6-NEXT: s_cbranch_execz .LBB28_139023; GFX6-NEXT: ; %bb.10:9024; GFX6-NEXT: v_mov_b32_e32 v3, s29025; GFX6-NEXT: s_mov_b32 m0, -19026; GFX6-NEXT: ds_read_b32 v2, v39027; GFX6-NEXT: s_mov_b64 s[2:3], 09028; GFX6-NEXT: .LBB28_11: ; %atomicrmw.start89029; GFX6-NEXT: ; =>This Inner Loop Header: Depth=19030; GFX6-NEXT: s_waitcnt lgkmcnt(0)9031; GFX6-NEXT: v_mov_b32_e32 v4, v29032; GFX6-NEXT: v_add_f32_e32 v2, v4, v19033; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v3, v4, v29034; GFX6-NEXT: s_waitcnt lgkmcnt(0)9035; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v49036; GFX6-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]9037; GFX6-NEXT: s_andn2_b64 exec, exec, s[2:3]9038; GFX6-NEXT: s_cbranch_execnz .LBB28_119039; GFX6-NEXT: ; %bb.12: ; %Flow9040; GFX6-NEXT: s_or_b64 exec, exec, s[2:3]9041; GFX6-NEXT: .LBB28_13: ; %Flow179042; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]9043; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x09044; GFX6-NEXT: v_readfirstlane_b32 s4, v29045; GFX6-NEXT: v_add_f32_e32 v0, s4, v09046; GFX6-NEXT: v_mov_b32_e32 v1, s49047; GFX6-NEXT: s_mov_b32 s3, 0xf0009048; GFX6-NEXT: s_mov_b32 s2, -19049; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc9050; GFX6-NEXT: s_waitcnt lgkmcnt(0)9051; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 09052; GFX6-NEXT: s_endpgm9053 %idx.add = add nuw i32 %idx, 49054 %shl0 = shl i32 %idx.add, 39055 %shl1 = shl i32 %idx.add, 49056 %ptr0 = inttoptr i32 %shl0 to ptr addrspace(3)9057 %ptr1 = inttoptr i32 %shl1 to ptr addrspace(3)9058 %a1 = atomicrmw fadd ptr addrspace(3) %ptr0, float 4.2e+1 seq_cst9059 %a2 = atomicrmw fadd ptr addrspace(3) %ptr1, float 4.2e+1 seq_cst9060 %a3 = atomicrmw fadd ptr addrspace(3) %ptrf, float %a1 seq_cst9061 store float %a3, ptr addrspace(1) %out9062 ret void9063}9064 9065define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspace(3) %ptrf, i32 %idx) {9066; GFX12-LABEL: local_ds_fadd_one_as:9067; GFX12: ; %bb.0:9068; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x89069; GFX12-NEXT: s_mov_b32 s6, exec_lo9070; GFX12-NEXT: ; implicit-def: $vgpr19071; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)9072; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 09073; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v09074; GFX12-NEXT: s_wait_kmcnt 0x09075; GFX12-NEXT: s_add_co_i32 s1, s3, 49076; GFX12-NEXT: s_and_saveexec_b32 s0, vcc_lo9077; GFX12-NEXT: s_cbranch_execz .LBB29_29078; GFX12-NEXT: ; %bb.1:9079; GFX12-NEXT: s_bcnt1_i32_b32 s3, s69080; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)9081; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s39082; GFX12-NEXT: s_lshl_b32 s3, s1, 39083; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)9084; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v19085; GFX12-NEXT: ds_add_rtn_f32 v1, v2, v19086; GFX12-NEXT: .LBB29_2:9087; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s09088; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9089; GFX12-NEXT: s_mov_b32 s7, exec_lo9090; GFX12-NEXT: s_wait_dscnt 0x09091; GFX12-NEXT: v_readfirstlane_b32 s3, v19092; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)9093; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 09094; GFX12-NEXT: s_mov_b32 s6, exec_lo9095; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)9096; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v29097; GFX12-NEXT: s_cbranch_execz .LBB29_49098; GFX12-NEXT: ; %bb.3:9099; GFX12-NEXT: s_bcnt1_i32_b32 s0, s79100; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)9101; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s09102; GFX12-NEXT: s_lshl_b32 s0, s1, 49103; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)9104; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v19105; GFX12-NEXT: ds_add_f32 v2, v19106; GFX12-NEXT: .LBB29_4:9107; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)9108; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s69109; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v0, v09110; GFX12-NEXT: s_mov_b32 s1, exec_lo9111; GFX12-NEXT: s_brev_b32 s0, 19112; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9113; GFX12-NEXT: v_mul_f32_e32 v0, 0x42280000, v09114; GFX12-NEXT: v_add_f32_e32 v0, s3, v09115; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)9116; GFX12-NEXT: v_cndmask_b32_e64 v1, v0, s3, vcc_lo9117; GFX12-NEXT: ; implicit-def: $vgpr09118; GFX12-NEXT: .LBB29_5: ; %ComputeLoop9119; GFX12-NEXT: ; =>This Inner Loop Header: Depth=19120; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)9121; GFX12-NEXT: s_ctz_i32_b32 s3, s19122; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)9123; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)9124; GFX12-NEXT: v_readlane_b32 s6, v1, s39125; GFX12-NEXT: v_writelane_b32 v0, s0, s39126; GFX12-NEXT: s_lshl_b32 s3, 1, s39127; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)9128; GFX12-NEXT: s_and_not1_b32 s1, s1, s39129; GFX12-NEXT: s_add_f32 s0, s0, s69130; GFX12-NEXT: s_cbranch_scc1 .LBB29_59131; GFX12-NEXT: ; %bb.6: ; %ComputeEnd9132; GFX12-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 09133; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)9134; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v19135; GFX12-NEXT: ; implicit-def: $vgpr19136; GFX12-NEXT: s_and_saveexec_b32 s1, vcc_lo9137; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)9138; GFX12-NEXT: s_xor_b32 s1, exec_lo, s19139; GFX12-NEXT: ; %bb.7:9140; GFX12-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s09141; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v29142; GFX12-NEXT: ; %bb.8:9143; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)9144; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s19145; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x09146; GFX12-NEXT: s_wait_dscnt 0x09147; GFX12-NEXT: v_readfirstlane_b32 s2, v19148; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)9149; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9150; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v09151; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo9152; GFX12-NEXT: s_wait_kmcnt 0x09153; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]9154; GFX12-NEXT: s_endpgm9155;9156; GFX942-LABEL: local_ds_fadd_one_as:9157; GFX942: ; %bb.0:9158; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x89159; GFX942-NEXT: s_mov_b64 s[0:1], exec9160; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 09161; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v09162; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v09163; GFX942-NEXT: s_waitcnt lgkmcnt(0)9164; GFX942-NEXT: s_add_i32 s3, s3, 49165; GFX942-NEXT: ; implicit-def: $vgpr19166; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc9167; GFX942-NEXT: s_cbranch_execz .LBB29_29168; GFX942-NEXT: ; %bb.1:9169; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]9170; GFX942-NEXT: s_lshl_b32 s8, s3, 39171; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s09172; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v19173; GFX942-NEXT: v_mov_b32_e32 v2, s89174; GFX942-NEXT: ds_add_rtn_f32 v1, v2, v19175; GFX942-NEXT: .LBB29_2:9176; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]9177; GFX942-NEXT: s_mov_b64 s[8:9], exec9178; GFX942-NEXT: s_waitcnt lgkmcnt(0)9179; GFX942-NEXT: v_readfirstlane_b32 s10, v19180; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 09181; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v19182; GFX942-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v19183; GFX942-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]9184; GFX942-NEXT: s_cbranch_execz .LBB29_49185; GFX942-NEXT: ; %bb.3:9186; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[8:9]9187; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s09188; GFX942-NEXT: s_lshl_b32 s0, s3, 49189; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v19190; GFX942-NEXT: v_mov_b32_e32 v2, s09191; GFX942-NEXT: ds_add_f32 v2, v19192; GFX942-NEXT: .LBB29_4:9193; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]9194; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v0, v09195; GFX942-NEXT: v_mul_f32_e32 v0, 0x42280000, v09196; GFX942-NEXT: v_add_f32_e32 v0, s10, v09197; GFX942-NEXT: v_mov_b32_e32 v1, s109198; GFX942-NEXT: s_mov_b64 s[0:1], exec9199; GFX942-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc9200; GFX942-NEXT: v_bfrev_b32_e32 v1, 19201; GFX942-NEXT: ; implicit-def: $vgpr09202; GFX942-NEXT: .LBB29_5: ; %ComputeLoop9203; GFX942-NEXT: ; =>This Inner Loop Header: Depth=19204; GFX942-NEXT: s_ff1_i32_b64 s3, s[0:1]9205; GFX942-NEXT: v_readfirstlane_b32 s6, v19206; GFX942-NEXT: s_mov_b32 m0, s39207; GFX942-NEXT: v_readlane_b32 s8, v2, s39208; GFX942-NEXT: v_writelane_b32 v0, s6, m09209; GFX942-NEXT: s_lshl_b64 s[6:7], 1, s39210; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]9211; GFX942-NEXT: v_add_f32_e32 v1, s8, v19212; GFX942-NEXT: s_cbranch_scc1 .LBB29_59213; GFX942-NEXT: ; %bb.6: ; %ComputeEnd9214; GFX942-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 09215; GFX942-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v29216; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v29217; GFX942-NEXT: ; implicit-def: $vgpr29218; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc9219; GFX942-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9220; GFX942-NEXT: ; %bb.7:9221; GFX942-NEXT: v_mov_b32_e32 v2, s29222; GFX942-NEXT: ds_add_rtn_f32 v2, v2, v19223; GFX942-NEXT: ; %bb.8:9224; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]9225; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x09226; GFX942-NEXT: s_waitcnt lgkmcnt(0)9227; GFX942-NEXT: v_readfirstlane_b32 s2, v29228; GFX942-NEXT: v_mov_b32_e32 v1, 09229; GFX942-NEXT: s_nop 09230; GFX942-NEXT: v_add_f32_e32 v0, s2, v09231; GFX942-NEXT: v_mov_b32_e32 v2, s29232; GFX942-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc9233; GFX942-NEXT: global_store_dword v1, v0, s[0:1]9234; GFX942-NEXT: s_endpgm9235;9236; GFX11-LABEL: local_ds_fadd_one_as:9237; GFX11: ; %bb.0:9238; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x89239; GFX11-NEXT: s_mov_b32 s6, exec_lo9240; GFX11-NEXT: ; implicit-def: $vgpr19241; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)9242; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 09243; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v09244; GFX11-NEXT: s_waitcnt lgkmcnt(0)9245; GFX11-NEXT: s_add_i32 s1, s3, 49246; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo9247; GFX11-NEXT: s_cbranch_execz .LBB29_29248; GFX11-NEXT: ; %bb.1:9249; GFX11-NEXT: s_bcnt1_i32_b32 s3, s69250; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9251; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s39252; GFX11-NEXT: s_lshl_b32 s3, s1, 39253; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)9254; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v19255; GFX11-NEXT: ds_add_rtn_f32 v1, v2, v19256; GFX11-NEXT: .LBB29_2:9257; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s09258; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)9259; GFX11-NEXT: s_mov_b32 s7, exec_lo9260; GFX11-NEXT: s_waitcnt lgkmcnt(0)9261; GFX11-NEXT: v_readfirstlane_b32 s3, v19262; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 09263; GFX11-NEXT: s_mov_b32 s6, exec_lo9264; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v29265; GFX11-NEXT: s_cbranch_execz .LBB29_49266; GFX11-NEXT: ; %bb.3:9267; GFX11-NEXT: s_bcnt1_i32_b32 s0, s79268; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)9269; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s09270; GFX11-NEXT: s_lshl_b32 s0, s1, 49271; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)9272; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v19273; GFX11-NEXT: ds_add_f32 v2, v19274; GFX11-NEXT: .LBB29_4:9275; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s69276; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v09277; GFX11-NEXT: v_bfrev_b32_e32 v1, 19278; GFX11-NEXT: s_mov_b32 s0, exec_lo9279; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)9280; GFX11-NEXT: v_mul_f32_e32 v0, 0x42280000, v09281; GFX11-NEXT: v_add_f32_e32 v0, s3, v09282; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)9283; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, s3, vcc_lo9284; GFX11-NEXT: ; implicit-def: $vgpr09285; GFX11-NEXT: .LBB29_5: ; %ComputeLoop9286; GFX11-NEXT: ; =>This Inner Loop Header: Depth=19287; GFX11-NEXT: s_ctz_i32_b32 s1, s09288; GFX11-NEXT: v_readfirstlane_b32 s3, v19289; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)9290; GFX11-NEXT: v_readlane_b32 s6, v2, s19291; GFX11-NEXT: v_writelane_b32 v0, s3, s19292; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9293; GFX11-NEXT: v_add_f32_e32 v1, s6, v19294; GFX11-NEXT: s_lshl_b32 s1, 1, s19295; GFX11-NEXT: s_and_not1_b32 s0, s0, s19296; GFX11-NEXT: s_cbranch_scc1 .LBB29_59297; GFX11-NEXT: ; %bb.6: ; %ComputeEnd9298; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 09299; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)9300; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v29301; GFX11-NEXT: ; implicit-def: $vgpr29302; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo9303; GFX11-NEXT: s_xor_b32 s0, exec_lo, s09304; GFX11-NEXT: ; %bb.7:9305; GFX11-NEXT: v_mov_b32_e32 v2, s29306; GFX11-NEXT: ds_add_rtn_f32 v2, v2, v19307; GFX11-NEXT: ; %bb.8:9308; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s09309; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x09310; GFX11-NEXT: s_waitcnt lgkmcnt(0)9311; GFX11-NEXT: v_readfirstlane_b32 s2, v29312; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)9313; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v09314; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo9315; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]9316; GFX11-NEXT: s_endpgm9317;9318; GFX10-LABEL: local_ds_fadd_one_as:9319; GFX10: ; %bb.0:9320; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x89321; GFX10-NEXT: s_mov_b32 s6, exec_lo9322; GFX10-NEXT: ; implicit-def: $vgpr19323; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 09324; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v09325; GFX10-NEXT: s_waitcnt lgkmcnt(0)9326; GFX10-NEXT: s_add_i32 s1, s3, 49327; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo9328; GFX10-NEXT: s_cbranch_execz .LBB29_29329; GFX10-NEXT: ; %bb.1:9330; GFX10-NEXT: s_bcnt1_i32_b32 s3, s69331; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s39332; GFX10-NEXT: s_lshl_b32 s3, s1, 39333; GFX10-NEXT: v_mov_b32_e32 v2, s39334; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v19335; GFX10-NEXT: ds_add_rtn_f32 v1, v2, v19336; GFX10-NEXT: .LBB29_2:9337; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)9338; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s09339; GFX10-NEXT: s_mov_b32 s7, exec_lo9340; GFX10-NEXT: s_waitcnt lgkmcnt(0)9341; GFX10-NEXT: v_readfirstlane_b32 s3, v19342; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 09343; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v29344; GFX10-NEXT: s_and_saveexec_b32 s6, s09345; GFX10-NEXT: s_cbranch_execz .LBB29_49346; GFX10-NEXT: ; %bb.3:9347; GFX10-NEXT: s_bcnt1_i32_b32 s0, s79348; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s09349; GFX10-NEXT: s_lshl_b32 s0, s1, 49350; GFX10-NEXT: v_mov_b32_e32 v2, s09351; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v19352; GFX10-NEXT: ds_add_f32 v2, v19353; GFX10-NEXT: .LBB29_4:9354; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)9355; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s69356; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v09357; GFX10-NEXT: v_bfrev_b32_e32 v1, 19358; GFX10-NEXT: s_mov_b32 s0, exec_lo9359; GFX10-NEXT: v_mul_f32_e32 v0, 0x42280000, v09360; GFX10-NEXT: v_add_f32_e32 v0, s3, v09361; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, s3, vcc_lo9362; GFX10-NEXT: ; implicit-def: $vgpr09363; GFX10-NEXT: .LBB29_5: ; %ComputeLoop9364; GFX10-NEXT: ; =>This Inner Loop Header: Depth=19365; GFX10-NEXT: s_ff1_i32_b32 s1, s09366; GFX10-NEXT: v_readfirstlane_b32 s3, v19367; GFX10-NEXT: v_readlane_b32 s6, v2, s19368; GFX10-NEXT: v_writelane_b32 v0, s3, s19369; GFX10-NEXT: v_add_f32_e32 v1, s6, v19370; GFX10-NEXT: s_lshl_b32 s1, 1, s19371; GFX10-NEXT: s_andn2_b32 s0, s0, s19372; GFX10-NEXT: s_cbranch_scc1 .LBB29_59373; GFX10-NEXT: ; %bb.6: ; %ComputeEnd9374; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 09375; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v29376; GFX10-NEXT: ; implicit-def: $vgpr29377; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo9378; GFX10-NEXT: s_xor_b32 s0, exec_lo, s09379; GFX10-NEXT: ; %bb.7:9380; GFX10-NEXT: v_mov_b32_e32 v2, s29381; GFX10-NEXT: ds_add_rtn_f32 v2, v2, v19382; GFX10-NEXT: ; %bb.8:9383; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)9384; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s09385; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x09386; GFX10-NEXT: s_waitcnt lgkmcnt(0)9387; GFX10-NEXT: v_readfirstlane_b32 s2, v29388; GFX10-NEXT: v_mov_b32_e32 v1, 09389; GFX10-NEXT: v_add_f32_e32 v0, s2, v09390; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo9391; GFX10-NEXT: global_store_dword v1, v0, s[0:1]9392; GFX10-NEXT: s_endpgm9393;9394; GFX90A-LABEL: local_ds_fadd_one_as:9395; GFX90A: ; %bb.0:9396; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x89397; GFX90A-NEXT: s_mov_b64 s[0:1], exec9398; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 09399; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v09400; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v09401; GFX90A-NEXT: s_waitcnt lgkmcnt(0)9402; GFX90A-NEXT: s_add_i32 s3, s3, 49403; GFX90A-NEXT: ; implicit-def: $vgpr19404; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc9405; GFX90A-NEXT: s_cbranch_execz .LBB29_29406; GFX90A-NEXT: ; %bb.1:9407; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]9408; GFX90A-NEXT: s_lshl_b32 s8, s3, 39409; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s09410; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v19411; GFX90A-NEXT: v_mov_b32_e32 v2, s89412; GFX90A-NEXT: ds_add_rtn_f32 v1, v2, v19413; GFX90A-NEXT: .LBB29_2:9414; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]9415; GFX90A-NEXT: s_mov_b64 s[8:9], exec9416; GFX90A-NEXT: s_waitcnt lgkmcnt(0)9417; GFX90A-NEXT: v_readfirstlane_b32 s10, v19418; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 09419; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v19420; GFX90A-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v19421; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]9422; GFX90A-NEXT: s_cbranch_execz .LBB29_49423; GFX90A-NEXT: ; %bb.3:9424; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[8:9]9425; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s09426; GFX90A-NEXT: s_lshl_b32 s0, s3, 49427; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v19428; GFX90A-NEXT: v_mov_b32_e32 v2, s09429; GFX90A-NEXT: ds_add_f32 v2, v19430; GFX90A-NEXT: .LBB29_4:9431; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]9432; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v09433; GFX90A-NEXT: v_mul_f32_e32 v0, 0x42280000, v09434; GFX90A-NEXT: v_add_f32_e32 v0, s10, v09435; GFX90A-NEXT: v_mov_b32_e32 v1, s109436; GFX90A-NEXT: s_mov_b64 s[0:1], exec9437; GFX90A-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc9438; GFX90A-NEXT: v_bfrev_b32_e32 v1, 19439; GFX90A-NEXT: ; implicit-def: $vgpr09440; GFX90A-NEXT: .LBB29_5: ; %ComputeLoop9441; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=19442; GFX90A-NEXT: s_ff1_i32_b64 s3, s[0:1]9443; GFX90A-NEXT: v_readfirstlane_b32 s6, v19444; GFX90A-NEXT: s_mov_b32 m0, s39445; GFX90A-NEXT: v_readlane_b32 s8, v2, s39446; GFX90A-NEXT: v_writelane_b32 v0, s6, m09447; GFX90A-NEXT: s_lshl_b64 s[6:7], 1, s39448; GFX90A-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]9449; GFX90A-NEXT: v_add_f32_e32 v1, s8, v19450; GFX90A-NEXT: s_cbranch_scc1 .LBB29_59451; GFX90A-NEXT: ; %bb.6: ; %ComputeEnd9452; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 09453; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v29454; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v29455; GFX90A-NEXT: ; implicit-def: $vgpr29456; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc9457; GFX90A-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9458; GFX90A-NEXT: ; %bb.7:9459; GFX90A-NEXT: v_mov_b32_e32 v2, s29460; GFX90A-NEXT: ds_add_rtn_f32 v2, v2, v19461; GFX90A-NEXT: ; %bb.8:9462; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]9463; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x09464; GFX90A-NEXT: s_waitcnt lgkmcnt(0)9465; GFX90A-NEXT: v_readfirstlane_b32 s2, v29466; GFX90A-NEXT: v_add_f32_e32 v0, s2, v09467; GFX90A-NEXT: v_mov_b32_e32 v2, s29468; GFX90A-NEXT: v_mov_b32_e32 v1, 09469; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc9470; GFX90A-NEXT: global_store_dword v1, v0, s[0:1]9471; GFX90A-NEXT: s_endpgm9472;9473; GFX908-LABEL: local_ds_fadd_one_as:9474; GFX908: ; %bb.0:9475; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x89476; GFX908-NEXT: s_mov_b64 s[0:1], exec9477; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 09478; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v09479; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v09480; GFX908-NEXT: s_waitcnt lgkmcnt(0)9481; GFX908-NEXT: s_add_i32 s3, s3, 49482; GFX908-NEXT: ; implicit-def: $vgpr19483; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc9484; GFX908-NEXT: s_cbranch_execz .LBB29_29485; GFX908-NEXT: ; %bb.1:9486; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]9487; GFX908-NEXT: s_lshl_b32 s8, s3, 39488; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s09489; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v19490; GFX908-NEXT: v_mov_b32_e32 v2, s89491; GFX908-NEXT: ds_add_rtn_f32 v1, v2, v19492; GFX908-NEXT: .LBB29_2:9493; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]9494; GFX908-NEXT: s_mov_b64 s[8:9], exec9495; GFX908-NEXT: s_waitcnt lgkmcnt(0)9496; GFX908-NEXT: v_readfirstlane_b32 s10, v19497; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 09498; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v19499; GFX908-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v19500; GFX908-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]9501; GFX908-NEXT: s_cbranch_execz .LBB29_49502; GFX908-NEXT: ; %bb.3:9503; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[8:9]9504; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s09505; GFX908-NEXT: s_lshl_b32 s0, s3, 49506; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v19507; GFX908-NEXT: v_mov_b32_e32 v2, s09508; GFX908-NEXT: ds_add_f32 v2, v19509; GFX908-NEXT: .LBB29_4:9510; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]9511; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v09512; GFX908-NEXT: v_mul_f32_e32 v0, 0x42280000, v09513; GFX908-NEXT: v_add_f32_e32 v0, s10, v09514; GFX908-NEXT: v_mov_b32_e32 v1, s109515; GFX908-NEXT: s_mov_b64 s[0:1], exec9516; GFX908-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc9517; GFX908-NEXT: v_bfrev_b32_e32 v1, 19518; GFX908-NEXT: ; implicit-def: $vgpr09519; GFX908-NEXT: .LBB29_5: ; %ComputeLoop9520; GFX908-NEXT: ; =>This Inner Loop Header: Depth=19521; GFX908-NEXT: s_ff1_i32_b64 s3, s[0:1]9522; GFX908-NEXT: v_readfirstlane_b32 s6, v19523; GFX908-NEXT: s_mov_b32 m0, s39524; GFX908-NEXT: v_readlane_b32 s8, v2, s39525; GFX908-NEXT: v_writelane_b32 v0, s6, m09526; GFX908-NEXT: s_lshl_b64 s[6:7], 1, s39527; GFX908-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]9528; GFX908-NEXT: v_add_f32_e32 v1, s8, v19529; GFX908-NEXT: s_cbranch_scc1 .LBB29_59530; GFX908-NEXT: ; %bb.6: ; %ComputeEnd9531; GFX908-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 09532; GFX908-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v29533; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v29534; GFX908-NEXT: ; implicit-def: $vgpr29535; GFX908-NEXT: s_and_saveexec_b64 s[0:1], vcc9536; GFX908-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9537; GFX908-NEXT: ; %bb.7:9538; GFX908-NEXT: v_mov_b32_e32 v2, s29539; GFX908-NEXT: ds_add_rtn_f32 v2, v2, v19540; GFX908-NEXT: ; %bb.8:9541; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]9542; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x09543; GFX908-NEXT: s_waitcnt lgkmcnt(0)9544; GFX908-NEXT: v_readfirstlane_b32 s2, v29545; GFX908-NEXT: v_add_f32_e32 v0, s2, v09546; GFX908-NEXT: v_mov_b32_e32 v2, s29547; GFX908-NEXT: v_mov_b32_e32 v1, 09548; GFX908-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc9549; GFX908-NEXT: global_store_dword v1, v0, s[0:1]9550; GFX908-NEXT: s_endpgm9551;9552; GFX8-LABEL: local_ds_fadd_one_as:9553; GFX8: ; %bb.0:9554; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x89555; GFX8-NEXT: s_mov_b64 s[0:1], exec9556; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 09557; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v09558; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v09559; GFX8-NEXT: s_waitcnt lgkmcnt(0)9560; GFX8-NEXT: s_add_i32 s3, s3, 49561; GFX8-NEXT: ; implicit-def: $vgpr19562; GFX8-NEXT: s_mov_b32 m0, -19563; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc9564; GFX8-NEXT: s_cbranch_execz .LBB29_29565; GFX8-NEXT: ; %bb.1:9566; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]9567; GFX8-NEXT: s_lshl_b32 s8, s3, 39568; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s09569; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v19570; GFX8-NEXT: v_mov_b32_e32 v2, s89571; GFX8-NEXT: ds_add_rtn_f32 v1, v2, v19572; GFX8-NEXT: .LBB29_2:9573; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]9574; GFX8-NEXT: s_mov_b64 s[8:9], exec9575; GFX8-NEXT: s_waitcnt lgkmcnt(0)9576; GFX8-NEXT: v_readfirstlane_b32 s10, v19577; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 09578; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v19579; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v19580; GFX8-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]9581; GFX8-NEXT: s_cbranch_execz .LBB29_49582; GFX8-NEXT: ; %bb.3:9583; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[8:9]9584; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s09585; GFX8-NEXT: s_lshl_b32 s0, s3, 49586; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v19587; GFX8-NEXT: v_mov_b32_e32 v2, s09588; GFX8-NEXT: ds_add_f32 v2, v19589; GFX8-NEXT: .LBB29_4:9590; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]9591; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v09592; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v09593; GFX8-NEXT: v_add_f32_e32 v0, s10, v09594; GFX8-NEXT: v_mov_b32_e32 v1, s109595; GFX8-NEXT: s_mov_b64 s[0:1], exec9596; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc9597; GFX8-NEXT: v_bfrev_b32_e32 v1, 19598; GFX8-NEXT: ; implicit-def: $vgpr09599; GFX8-NEXT: .LBB29_5: ; %ComputeLoop9600; GFX8-NEXT: ; =>This Inner Loop Header: Depth=19601; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]9602; GFX8-NEXT: v_readfirstlane_b32 s6, v19603; GFX8-NEXT: s_mov_b32 m0, s39604; GFX8-NEXT: v_readlane_b32 s8, v2, s39605; GFX8-NEXT: v_writelane_b32 v0, s6, m09606; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s39607; GFX8-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]9608; GFX8-NEXT: v_add_f32_e32 v1, s8, v19609; GFX8-NEXT: s_cbranch_scc1 .LBB29_59610; GFX8-NEXT: ; %bb.6: ; %ComputeEnd9611; GFX8-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 09612; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v29613; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v29614; GFX8-NEXT: ; implicit-def: $vgpr29615; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc9616; GFX8-NEXT: s_xor_b64 s[0:1], exec, s[0:1]9617; GFX8-NEXT: ; %bb.7:9618; GFX8-NEXT: v_mov_b32_e32 v2, s29619; GFX8-NEXT: s_mov_b32 m0, -19620; GFX8-NEXT: ds_add_rtn_f32 v2, v2, v19621; GFX8-NEXT: ; %bb.8:9622; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]9623; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x09624; GFX8-NEXT: s_waitcnt lgkmcnt(0)9625; GFX8-NEXT: v_readfirstlane_b32 s2, v29626; GFX8-NEXT: v_add_f32_e32 v0, s2, v09627; GFX8-NEXT: v_mov_b32_e32 v1, s29628; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc9629; GFX8-NEXT: v_mov_b32_e32 v0, s09630; GFX8-NEXT: v_mov_b32_e32 v1, s19631; GFX8-NEXT: flat_store_dword v[0:1], v29632; GFX8-NEXT: s_endpgm9633;9634; GFX7-LABEL: local_ds_fadd_one_as:9635; GFX7: ; %bb.0:9636; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x29637; GFX7-NEXT: s_mov_b64 s[0:1], exec9638; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 09639; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v09640; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v09641; GFX7-NEXT: s_waitcnt lgkmcnt(0)9642; GFX7-NEXT: s_add_i32 s3, s3, 49643; GFX7-NEXT: ; implicit-def: $vgpr19644; GFX7-NEXT: s_mov_b32 m0, -19645; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc9646; GFX7-NEXT: s_cbranch_execz .LBB29_49647; GFX7-NEXT: ; %bb.1:9648; GFX7-NEXT: s_lshl_b32 s8, s3, 39649; GFX7-NEXT: v_mov_b32_e32 v2, s89650; GFX7-NEXT: ds_read_b32 v1, v29651; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]9652; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s09653; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v39654; GFX7-NEXT: s_mov_b64 s[8:9], 09655; GFX7-NEXT: .LBB29_2: ; %atomicrmw.start9656; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19657; GFX7-NEXT: s_waitcnt lgkmcnt(0)9658; GFX7-NEXT: v_mov_b32_e32 v4, v19659; GFX7-NEXT: v_add_f32_e32 v1, v4, v39660; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v19661; GFX7-NEXT: s_waitcnt lgkmcnt(0)9662; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v49663; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]9664; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]9665; GFX7-NEXT: s_cbranch_execnz .LBB29_29666; GFX7-NEXT: ; %bb.3: ; %Flow229667; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]9668; GFX7-NEXT: .LBB29_4: ; %Flow239669; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]9670; GFX7-NEXT: s_mov_b64 s[8:9], exec9671; GFX7-NEXT: v_readfirstlane_b32 s10, v19672; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 09673; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v19674; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v19675; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]9676; GFX7-NEXT: s_cbranch_execz .LBB29_79677; GFX7-NEXT: ; %bb.5:9678; GFX7-NEXT: s_lshl_b32 s0, s3, 49679; GFX7-NEXT: v_mov_b32_e32 v1, s09680; GFX7-NEXT: ds_read_b32 v3, v19681; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[8:9]9682; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s09683; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v29684; GFX7-NEXT: s_mov_b64 s[8:9], 09685; GFX7-NEXT: .LBB29_6: ; %atomicrmw.start29686; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19687; GFX7-NEXT: s_waitcnt lgkmcnt(0)9688; GFX7-NEXT: v_add_f32_e32 v4, v3, v29689; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v49690; GFX7-NEXT: s_waitcnt lgkmcnt(0)9691; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v39692; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]9693; GFX7-NEXT: v_mov_b32_e32 v3, v49694; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]9695; GFX7-NEXT: s_cbranch_execnz .LBB29_69696; GFX7-NEXT: .LBB29_7: ; %Flow219697; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]9698; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v09699; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v09700; GFX7-NEXT: v_add_f32_e32 v0, s10, v09701; GFX7-NEXT: v_mov_b32_e32 v1, s109702; GFX7-NEXT: s_mov_b64 s[0:1], exec9703; GFX7-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc9704; GFX7-NEXT: v_bfrev_b32_e32 v1, 19705; GFX7-NEXT: ; implicit-def: $vgpr09706; GFX7-NEXT: .LBB29_8: ; %ComputeLoop9707; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19708; GFX7-NEXT: s_ff1_i32_b64 s3, s[0:1]9709; GFX7-NEXT: s_lshl_b64 s[6:7], 1, s39710; GFX7-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]9711; GFX7-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 09712; GFX7-NEXT: v_readfirstlane_b32 s8, v19713; GFX7-NEXT: v_readlane_b32 s9, v2, s39714; GFX7-NEXT: s_mov_b32 m0, s39715; GFX7-NEXT: v_writelane_b32 v0, s8, m09716; GFX7-NEXT: v_add_f32_e32 v1, s9, v19717; GFX7-NEXT: s_and_b64 vcc, exec, s[6:7]9718; GFX7-NEXT: s_cbranch_vccnz .LBB29_89719; GFX7-NEXT: ; %bb.9: ; %ComputeEnd9720; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 09721; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v29722; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v29723; GFX7-NEXT: ; implicit-def: $vgpr29724; GFX7-NEXT: s_and_saveexec_b64 s[0:1], vcc9725; GFX7-NEXT: s_xor_b64 s[6:7], exec, s[0:1]9726; GFX7-NEXT: s_cbranch_execz .LBB29_139727; GFX7-NEXT: ; %bb.10:9728; GFX7-NEXT: v_mov_b32_e32 v3, s29729; GFX7-NEXT: s_mov_b32 m0, -19730; GFX7-NEXT: ds_read_b32 v2, v39731; GFX7-NEXT: s_mov_b64 s[2:3], 09732; GFX7-NEXT: .LBB29_11: ; %atomicrmw.start89733; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19734; GFX7-NEXT: s_waitcnt lgkmcnt(0)9735; GFX7-NEXT: v_mov_b32_e32 v4, v29736; GFX7-NEXT: v_add_f32_e32 v2, v4, v19737; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v3, v4, v29738; GFX7-NEXT: s_waitcnt lgkmcnt(0)9739; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v49740; GFX7-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]9741; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]9742; GFX7-NEXT: s_cbranch_execnz .LBB29_119743; GFX7-NEXT: ; %bb.12: ; %Flow9744; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]9745; GFX7-NEXT: .LBB29_13: ; %Flow199746; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]9747; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x09748; GFX7-NEXT: v_readfirstlane_b32 s4, v29749; GFX7-NEXT: v_add_f32_e32 v0, s4, v09750; GFX7-NEXT: v_mov_b32_e32 v1, s49751; GFX7-NEXT: s_mov_b32 s3, 0xf0009752; GFX7-NEXT: s_mov_b32 s2, -19753; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc9754; GFX7-NEXT: s_waitcnt lgkmcnt(0)9755; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 09756; GFX7-NEXT: s_endpgm9757;9758; GFX6-LABEL: local_ds_fadd_one_as:9759; GFX6: ; %bb.0:9760; GFX6-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x29761; GFX6-NEXT: s_mov_b64 s[0:1], exec9762; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 09763; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v09764; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v09765; GFX6-NEXT: s_waitcnt lgkmcnt(0)9766; GFX6-NEXT: s_add_i32 s3, s3, 49767; GFX6-NEXT: ; implicit-def: $vgpr19768; GFX6-NEXT: s_mov_b32 m0, -19769; GFX6-NEXT: s_and_saveexec_b64 s[6:7], vcc9770; GFX6-NEXT: s_cbranch_execz .LBB29_49771; GFX6-NEXT: ; %bb.1:9772; GFX6-NEXT: s_lshl_b32 s8, s3, 39773; GFX6-NEXT: v_mov_b32_e32 v2, s89774; GFX6-NEXT: ds_read_b32 v1, v29775; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]9776; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v3, s09777; GFX6-NEXT: v_mul_f32_e32 v3, 0x42280000, v39778; GFX6-NEXT: s_mov_b64 s[8:9], 09779; GFX6-NEXT: .LBB29_2: ; %atomicrmw.start9780; GFX6-NEXT: ; =>This Inner Loop Header: Depth=19781; GFX6-NEXT: s_waitcnt lgkmcnt(0)9782; GFX6-NEXT: v_mov_b32_e32 v4, v19783; GFX6-NEXT: v_add_f32_e32 v1, v4, v39784; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v19785; GFX6-NEXT: s_waitcnt lgkmcnt(0)9786; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v49787; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]9788; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]9789; GFX6-NEXT: s_cbranch_execnz .LBB29_29790; GFX6-NEXT: ; %bb.3: ; %Flow209791; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]9792; GFX6-NEXT: .LBB29_4: ; %Flow219793; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]9794; GFX6-NEXT: s_mov_b64 s[8:9], exec9795; GFX6-NEXT: v_readfirstlane_b32 s10, v19796; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 09797; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v19798; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v19799; GFX6-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]9800; GFX6-NEXT: s_cbranch_execz .LBB29_79801; GFX6-NEXT: ; %bb.5:9802; GFX6-NEXT: s_lshl_b32 s0, s3, 49803; GFX6-NEXT: v_mov_b32_e32 v1, s09804; GFX6-NEXT: ds_read_b32 v3, v19805; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[8:9]9806; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s09807; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v29808; GFX6-NEXT: s_mov_b64 s[8:9], 09809; GFX6-NEXT: .LBB29_6: ; %atomicrmw.start29810; GFX6-NEXT: ; =>This Inner Loop Header: Depth=19811; GFX6-NEXT: s_waitcnt lgkmcnt(0)9812; GFX6-NEXT: v_add_f32_e32 v4, v3, v29813; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v49814; GFX6-NEXT: s_waitcnt lgkmcnt(0)9815; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v39816; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]9817; GFX6-NEXT: v_mov_b32_e32 v3, v49818; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]9819; GFX6-NEXT: s_cbranch_execnz .LBB29_69820; GFX6-NEXT: .LBB29_7: ; %Flow199821; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]9822; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, v09823; GFX6-NEXT: v_mul_f32_e32 v0, 0x42280000, v09824; GFX6-NEXT: v_add_f32_e32 v0, s10, v09825; GFX6-NEXT: v_mov_b32_e32 v1, s109826; GFX6-NEXT: s_mov_b64 s[0:1], exec9827; GFX6-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc9828; GFX6-NEXT: v_bfrev_b32_e32 v1, 19829; GFX6-NEXT: ; implicit-def: $vgpr09830; GFX6-NEXT: .LBB29_8: ; %ComputeLoop9831; GFX6-NEXT: ; =>This Inner Loop Header: Depth=19832; GFX6-NEXT: s_ff1_i32_b64 s3, s[0:1]9833; GFX6-NEXT: s_lshl_b64 s[6:7], 1, s39834; GFX6-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]9835; GFX6-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 09836; GFX6-NEXT: v_readfirstlane_b32 s8, v19837; GFX6-NEXT: v_readlane_b32 s9, v2, s39838; GFX6-NEXT: s_mov_b32 m0, s39839; GFX6-NEXT: v_writelane_b32 v0, s8, m09840; GFX6-NEXT: v_add_f32_e32 v1, s9, v19841; GFX6-NEXT: s_and_b64 vcc, exec, s[6:7]9842; GFX6-NEXT: s_cbranch_vccnz .LBB29_89843; GFX6-NEXT: ; %bb.9: ; %ComputeEnd9844; GFX6-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 09845; GFX6-NEXT: v_mbcnt_hi_u32_b32_e32 v2, exec_hi, v29846; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v29847; GFX6-NEXT: ; implicit-def: $vgpr29848; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc9849; GFX6-NEXT: s_xor_b64 s[6:7], exec, s[0:1]9850; GFX6-NEXT: s_cbranch_execz .LBB29_139851; GFX6-NEXT: ; %bb.10:9852; GFX6-NEXT: v_mov_b32_e32 v3, s29853; GFX6-NEXT: s_mov_b32 m0, -19854; GFX6-NEXT: ds_read_b32 v2, v39855; GFX6-NEXT: s_mov_b64 s[2:3], 09856; GFX6-NEXT: .LBB29_11: ; %atomicrmw.start89857; GFX6-NEXT: ; =>This Inner Loop Header: Depth=19858; GFX6-NEXT: s_waitcnt lgkmcnt(0)9859; GFX6-NEXT: v_mov_b32_e32 v4, v29860; GFX6-NEXT: v_add_f32_e32 v2, v4, v19861; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v3, v4, v29862; GFX6-NEXT: s_waitcnt lgkmcnt(0)9863; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v49864; GFX6-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]9865; GFX6-NEXT: s_andn2_b64 exec, exec, s[2:3]9866; GFX6-NEXT: s_cbranch_execnz .LBB29_119867; GFX6-NEXT: ; %bb.12: ; %Flow9868; GFX6-NEXT: s_or_b64 exec, exec, s[2:3]9869; GFX6-NEXT: .LBB29_13: ; %Flow179870; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]9871; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x09872; GFX6-NEXT: v_readfirstlane_b32 s4, v29873; GFX6-NEXT: v_add_f32_e32 v0, s4, v09874; GFX6-NEXT: v_mov_b32_e32 v1, s49875; GFX6-NEXT: s_mov_b32 s3, 0xf0009876; GFX6-NEXT: s_mov_b32 s2, -19877; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc9878; GFX6-NEXT: s_waitcnt lgkmcnt(0)9879; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 09880; GFX6-NEXT: s_endpgm9881 %idx.add = add nuw i32 %idx, 49882 %shl0 = shl i32 %idx.add, 39883 %shl1 = shl i32 %idx.add, 49884 %ptr0 = inttoptr i32 %shl0 to ptr addrspace(3)9885 %ptr1 = inttoptr i32 %shl1 to ptr addrspace(3)9886 %a1 = atomicrmw fadd ptr addrspace(3) %ptr0, float 4.2e+1 syncscope("one-as") seq_cst9887 %a2 = atomicrmw fadd ptr addrspace(3) %ptr1, float 4.2e+1 syncscope("one-as") seq_cst9888 %a3 = atomicrmw fadd ptr addrspace(3) %ptrf, float %a1 syncscope("one-as") seq_cst9889 store float %a3, ptr addrspace(1) %out9890 ret void9891}9892 9893define float @local_atomic_fadd_ret_f32__amdgpu_ignore_denormal_mode(ptr addrspace(3) %ptr) {9894; GFX12-LABEL: local_atomic_fadd_ret_f32__amdgpu_ignore_denormal_mode:9895; GFX12: ; %bb.0:9896; GFX12-NEXT: s_wait_loadcnt_dscnt 0x09897; GFX12-NEXT: s_wait_expcnt 0x09898; GFX12-NEXT: s_wait_samplecnt 0x09899; GFX12-NEXT: s_wait_bvhcnt 0x09900; GFX12-NEXT: s_wait_kmcnt 0x09901; GFX12-NEXT: v_mov_b32_e32 v1, 4.09902; GFX12-NEXT: s_wait_storecnt 0x09903; GFX12-NEXT: ds_add_rtn_f32 v0, v0, v19904; GFX12-NEXT: s_wait_dscnt 0x09905; GFX12-NEXT: global_inv scope:SCOPE_SE9906; GFX12-NEXT: s_wait_loadcnt 0x09907; GFX12-NEXT: s_setpc_b64 s[30:31]9908;9909; GFX942-LABEL: local_atomic_fadd_ret_f32__amdgpu_ignore_denormal_mode:9910; GFX942: ; %bb.0:9911; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9912; GFX942-NEXT: v_mov_b32_e32 v1, 4.09913; GFX942-NEXT: ds_add_rtn_f32 v0, v0, v19914; GFX942-NEXT: s_waitcnt lgkmcnt(0)9915; GFX942-NEXT: s_setpc_b64 s[30:31]9916;9917; GFX11-LABEL: local_atomic_fadd_ret_f32__amdgpu_ignore_denormal_mode:9918; GFX11: ; %bb.0:9919; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9920; GFX11-NEXT: v_mov_b32_e32 v1, 4.09921; GFX11-NEXT: s_waitcnt_vscnt null, 0x09922; GFX11-NEXT: ds_add_rtn_f32 v0, v0, v19923; GFX11-NEXT: s_waitcnt lgkmcnt(0)9924; GFX11-NEXT: buffer_gl0_inv9925; GFX11-NEXT: s_setpc_b64 s[30:31]9926;9927; GFX10-LABEL: local_atomic_fadd_ret_f32__amdgpu_ignore_denormal_mode:9928; GFX10: ; %bb.0:9929; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9930; GFX10-NEXT: v_mov_b32_e32 v1, 4.09931; GFX10-NEXT: s_waitcnt_vscnt null, 0x09932; GFX10-NEXT: ds_add_rtn_f32 v0, v0, v19933; GFX10-NEXT: s_waitcnt lgkmcnt(0)9934; GFX10-NEXT: buffer_gl0_inv9935; GFX10-NEXT: s_setpc_b64 s[30:31]9936;9937; GFX90A-LABEL: local_atomic_fadd_ret_f32__amdgpu_ignore_denormal_mode:9938; GFX90A: ; %bb.0:9939; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9940; GFX90A-NEXT: v_mov_b32_e32 v1, 4.09941; GFX90A-NEXT: ds_add_rtn_f32 v0, v0, v19942; GFX90A-NEXT: s_waitcnt lgkmcnt(0)9943; GFX90A-NEXT: s_setpc_b64 s[30:31]9944;9945; GFX908-LABEL: local_atomic_fadd_ret_f32__amdgpu_ignore_denormal_mode:9946; GFX908: ; %bb.0:9947; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9948; GFX908-NEXT: v_mov_b32_e32 v1, 4.09949; GFX908-NEXT: ds_add_rtn_f32 v0, v0, v19950; GFX908-NEXT: s_waitcnt lgkmcnt(0)9951; GFX908-NEXT: s_setpc_b64 s[30:31]9952;9953; GFX8-LABEL: local_atomic_fadd_ret_f32__amdgpu_ignore_denormal_mode:9954; GFX8: ; %bb.0:9955; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9956; GFX8-NEXT: v_mov_b32_e32 v1, 4.09957; GFX8-NEXT: s_mov_b32 m0, -19958; GFX8-NEXT: ds_add_rtn_f32 v0, v0, v19959; GFX8-NEXT: s_waitcnt lgkmcnt(0)9960; GFX8-NEXT: s_setpc_b64 s[30:31]9961;9962; GFX7-LABEL: local_atomic_fadd_ret_f32__amdgpu_ignore_denormal_mode:9963; GFX7: ; %bb.0:9964; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9965; GFX7-NEXT: s_mov_b32 m0, -19966; GFX7-NEXT: ds_read_b32 v1, v09967; GFX7-NEXT: s_mov_b64 s[4:5], 09968; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start9969; GFX7-NEXT: ; =>This Inner Loop Header: Depth=19970; GFX7-NEXT: s_waitcnt lgkmcnt(0)9971; GFX7-NEXT: v_mov_b32_e32 v2, v19972; GFX7-NEXT: v_add_f32_e32 v1, 4.0, v29973; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v19974; GFX7-NEXT: s_waitcnt lgkmcnt(0)9975; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v29976; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9977; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]9978; GFX7-NEXT: s_cbranch_execnz .LBB30_19979; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end9980; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]9981; GFX7-NEXT: v_mov_b32_e32 v0, v19982; GFX7-NEXT: s_setpc_b64 s[30:31]9983;9984; GFX6-LABEL: local_atomic_fadd_ret_f32__amdgpu_ignore_denormal_mode:9985; GFX6: ; %bb.0:9986; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9987; GFX6-NEXT: s_mov_b32 m0, -19988; GFX6-NEXT: ds_read_b32 v1, v09989; GFX6-NEXT: s_mov_b64 s[4:5], 09990; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start9991; GFX6-NEXT: ; =>This Inner Loop Header: Depth=19992; GFX6-NEXT: s_waitcnt lgkmcnt(0)9993; GFX6-NEXT: v_mov_b32_e32 v2, v19994; GFX6-NEXT: v_add_f32_e32 v1, 4.0, v29995; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v19996; GFX6-NEXT: s_waitcnt lgkmcnt(0)9997; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v29998; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]9999; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]10000; GFX6-NEXT: s_cbranch_execnz .LBB30_110001; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end10002; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]10003; GFX6-NEXT: v_mov_b32_e32 v0, v110004; GFX6-NEXT: s_setpc_b64 s[30:31]10005 %result = atomicrmw fadd ptr addrspace(3) %ptr, float 4.0 seq_cst, !amdgpu.ignore.denormal.mode !010006 ret float %result10007}10008 10009define void @local_atomic_fadd_noret_f32__amdgpu_ignore_denormal_mode(ptr addrspace(3) %ptr) {10010; GFX12-LABEL: local_atomic_fadd_noret_f32__amdgpu_ignore_denormal_mode:10011; GFX12: ; %bb.0:10012; GFX12-NEXT: s_wait_loadcnt_dscnt 0x010013; GFX12-NEXT: s_wait_expcnt 0x010014; GFX12-NEXT: s_wait_samplecnt 0x010015; GFX12-NEXT: s_wait_bvhcnt 0x010016; GFX12-NEXT: s_wait_kmcnt 0x010017; GFX12-NEXT: v_mov_b32_e32 v1, 4.010018; GFX12-NEXT: s_wait_storecnt 0x010019; GFX12-NEXT: ds_add_f32 v0, v110020; GFX12-NEXT: s_wait_dscnt 0x010021; GFX12-NEXT: global_inv scope:SCOPE_SE10022; GFX12-NEXT: s_wait_loadcnt 0x010023; GFX12-NEXT: s_setpc_b64 s[30:31]10024;10025; GFX942-LABEL: local_atomic_fadd_noret_f32__amdgpu_ignore_denormal_mode:10026; GFX942: ; %bb.0:10027; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10028; GFX942-NEXT: v_mov_b32_e32 v1, 4.010029; GFX942-NEXT: ds_add_f32 v0, v110030; GFX942-NEXT: s_waitcnt lgkmcnt(0)10031; GFX942-NEXT: s_setpc_b64 s[30:31]10032;10033; GFX11-LABEL: local_atomic_fadd_noret_f32__amdgpu_ignore_denormal_mode:10034; GFX11: ; %bb.0:10035; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10036; GFX11-NEXT: v_mov_b32_e32 v1, 4.010037; GFX11-NEXT: s_waitcnt_vscnt null, 0x010038; GFX11-NEXT: ds_add_f32 v0, v110039; GFX11-NEXT: s_waitcnt lgkmcnt(0)10040; GFX11-NEXT: buffer_gl0_inv10041; GFX11-NEXT: s_setpc_b64 s[30:31]10042;10043; GFX10-LABEL: local_atomic_fadd_noret_f32__amdgpu_ignore_denormal_mode:10044; GFX10: ; %bb.0:10045; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10046; GFX10-NEXT: v_mov_b32_e32 v1, 4.010047; GFX10-NEXT: s_waitcnt_vscnt null, 0x010048; GFX10-NEXT: ds_add_f32 v0, v110049; GFX10-NEXT: s_waitcnt lgkmcnt(0)10050; GFX10-NEXT: buffer_gl0_inv10051; GFX10-NEXT: s_setpc_b64 s[30:31]10052;10053; GFX90A-LABEL: local_atomic_fadd_noret_f32__amdgpu_ignore_denormal_mode:10054; GFX90A: ; %bb.0:10055; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10056; GFX90A-NEXT: v_mov_b32_e32 v1, 4.010057; GFX90A-NEXT: ds_add_f32 v0, v110058; GFX90A-NEXT: s_waitcnt lgkmcnt(0)10059; GFX90A-NEXT: s_setpc_b64 s[30:31]10060;10061; GFX908-LABEL: local_atomic_fadd_noret_f32__amdgpu_ignore_denormal_mode:10062; GFX908: ; %bb.0:10063; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10064; GFX908-NEXT: v_mov_b32_e32 v1, 4.010065; GFX908-NEXT: ds_add_f32 v0, v110066; GFX908-NEXT: s_waitcnt lgkmcnt(0)10067; GFX908-NEXT: s_setpc_b64 s[30:31]10068;10069; GFX8-LABEL: local_atomic_fadd_noret_f32__amdgpu_ignore_denormal_mode:10070; GFX8: ; %bb.0:10071; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10072; GFX8-NEXT: v_mov_b32_e32 v1, 4.010073; GFX8-NEXT: s_mov_b32 m0, -110074; GFX8-NEXT: ds_add_f32 v0, v110075; GFX8-NEXT: s_waitcnt lgkmcnt(0)10076; GFX8-NEXT: s_setpc_b64 s[30:31]10077;10078; GFX7-LABEL: local_atomic_fadd_noret_f32__amdgpu_ignore_denormal_mode:10079; GFX7: ; %bb.0:10080; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10081; GFX7-NEXT: s_mov_b32 m0, -110082; GFX7-NEXT: ds_read_b32 v1, v010083; GFX7-NEXT: s_mov_b64 s[4:5], 010084; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start10085; GFX7-NEXT: ; =>This Inner Loop Header: Depth=110086; GFX7-NEXT: s_waitcnt lgkmcnt(0)10087; GFX7-NEXT: v_add_f32_e32 v2, 4.0, v110088; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v210089; GFX7-NEXT: s_waitcnt lgkmcnt(0)10090; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v110091; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10092; GFX7-NEXT: v_mov_b32_e32 v1, v210093; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]10094; GFX7-NEXT: s_cbranch_execnz .LBB31_110095; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end10096; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]10097; GFX7-NEXT: s_setpc_b64 s[30:31]10098;10099; GFX6-LABEL: local_atomic_fadd_noret_f32__amdgpu_ignore_denormal_mode:10100; GFX6: ; %bb.0:10101; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)10102; GFX6-NEXT: s_mov_b32 m0, -110103; GFX6-NEXT: ds_read_b32 v1, v010104; GFX6-NEXT: s_mov_b64 s[4:5], 010105; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start10106; GFX6-NEXT: ; =>This Inner Loop Header: Depth=110107; GFX6-NEXT: s_waitcnt lgkmcnt(0)10108; GFX6-NEXT: v_add_f32_e32 v2, 4.0, v110109; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v210110; GFX6-NEXT: s_waitcnt lgkmcnt(0)10111; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v110112; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]10113; GFX6-NEXT: v_mov_b32_e32 v1, v210114; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]10115; GFX6-NEXT: s_cbranch_execnz .LBB31_110116; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end10117; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]10118; GFX6-NEXT: s_setpc_b64 s[30:31]10119 %result = atomicrmw fadd ptr addrspace(3) %ptr, float 4.0 seq_cst, !amdgpu.ignore.denormal.mode !010120 ret void10121}10122 10123!0 = !{}10124