467 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s3; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s4; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s5; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s6 7define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) #0 {8; GFX90A-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:9; GFX90A: ; %bb.0:10; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)11; GFX90A-NEXT: v_mov_b32_e32 v3, v212; GFX90A-NEXT: v_mov_b32_e32 v2, v113; GFX90A-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[16:19], s20 idxen offen glc14; GFX90A-NEXT: s_waitcnt vmcnt(0)15; GFX90A-NEXT: s_setpc_b64 s[30:31]16;17; GFX942-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:18; GFX942: ; %bb.0:19; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)20; GFX942-NEXT: v_mov_b32_e32 v3, v221; GFX942-NEXT: v_mov_b32_e32 v2, v122; GFX942-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen sc023; GFX942-NEXT: s_waitcnt vmcnt(0)24; GFX942-NEXT: s_setpc_b64 s[30:31]25;26; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:27; GFX1200: ; %bb.0:28; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x029; GFX1200-NEXT: s_wait_expcnt 0x030; GFX1200-NEXT: s_wait_samplecnt 0x031; GFX1200-NEXT: s_wait_bvhcnt 0x032; GFX1200-NEXT: s_wait_kmcnt 0x033; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN34; GFX1200-NEXT: s_wait_loadcnt 0x035; GFX1200-NEXT: s_setpc_b64 s[30:31]36;37; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:38; GFX1250: ; %bb.0:39; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x040; GFX1250-NEXT: s_wait_kmcnt 0x041; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v142; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN43; GFX1250-NEXT: s_wait_loadcnt 0x044; GFX1250-NEXT: s_set_pc_i64 s[30:31]45 %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)46 ret float %ret47}48 49; Natural mapping, no voffset50define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset(float %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 inreg %soffset) #0 {51; GFX90A-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:52; GFX90A: ; %bb.0:53; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)54; GFX90A-NEXT: buffer_atomic_add_f32 v0, v1, s[16:19], s20 idxen glc55; GFX90A-NEXT: s_waitcnt vmcnt(0)56; GFX90A-NEXT: s_setpc_b64 s[30:31]57;58; GFX942-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:59; GFX942: ; %bb.0:60; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)61; GFX942-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], s16 idxen sc062; GFX942-NEXT: s_waitcnt vmcnt(0)63; GFX942-NEXT: s_setpc_b64 s[30:31]64;65; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:66; GFX1200: ; %bb.0:67; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x068; GFX1200-NEXT: s_wait_expcnt 0x069; GFX1200-NEXT: s_wait_samplecnt 0x070; GFX1200-NEXT: s_wait_bvhcnt 0x071; GFX1200-NEXT: s_wait_kmcnt 0x072; GFX1200-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], s16 idxen th:TH_ATOMIC_RETURN73; GFX1200-NEXT: s_wait_loadcnt 0x074; GFX1200-NEXT: s_setpc_b64 s[30:31]75;76; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:77; GFX1250: ; %bb.0:78; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x079; GFX1250-NEXT: s_wait_kmcnt 0x080; GFX1250-NEXT: buffer_atomic_add_f32 v0, v1, s[0:3], s16 idxen th:TH_ATOMIC_RETURN81; GFX1250-NEXT: s_wait_loadcnt 0x082; GFX1250-NEXT: s_set_pc_i64 s[30:31]83 %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)84 ret float %ret85}86 87define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc(float %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) #0 {88; GFX90A-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:89; GFX90A: ; %bb.0:90; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)91; GFX90A-NEXT: v_mov_b32_e32 v3, v292; GFX90A-NEXT: v_mov_b32_e32 v2, v193; GFX90A-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[16:19], s20 idxen offen glc slc94; GFX90A-NEXT: s_waitcnt vmcnt(0)95; GFX90A-NEXT: s_setpc_b64 s[30:31]96;97; GFX942-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:98; GFX942: ; %bb.0:99; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)100; GFX942-NEXT: v_mov_b32_e32 v3, v2101; GFX942-NEXT: v_mov_b32_e32 v2, v1102; GFX942-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen sc0 nt103; GFX942-NEXT: s_waitcnt vmcnt(0)104; GFX942-NEXT: s_setpc_b64 s[30:31]105;106; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:107; GFX1200: ; %bb.0:108; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0109; GFX1200-NEXT: s_wait_expcnt 0x0110; GFX1200-NEXT: s_wait_samplecnt 0x0111; GFX1200-NEXT: s_wait_bvhcnt 0x0112; GFX1200-NEXT: s_wait_kmcnt 0x0113; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT_RETURN114; GFX1200-NEXT: s_wait_loadcnt 0x0115; GFX1200-NEXT: s_setpc_b64 s[30:31]116;117; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:118; GFX1250: ; %bb.0:119; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0120; GFX1250-NEXT: s_wait_kmcnt 0x0121; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1122; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_NT_RETURN123; GFX1250-NEXT: s_wait_loadcnt 0x0124; GFX1250-NEXT: s_set_pc_i64 s[30:31]125 %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)126 ret float %ret127}128 129define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<2 x half> %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) #0 {130; GFX90A-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:131; GFX90A: ; %bb.0:132; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)133; GFX90A-NEXT: v_mov_b32_e32 v3, v2134; GFX90A-NEXT: v_mov_b32_e32 v2, v1135; GFX90A-NEXT: buffer_atomic_pk_add_f16 v0, v[2:3], s[16:19], s20 idxen offen glc136; GFX90A-NEXT: s_waitcnt vmcnt(0)137; GFX90A-NEXT: s_setpc_b64 s[30:31]138;139; GFX942-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:140; GFX942: ; %bb.0:141; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)142; GFX942-NEXT: v_mov_b32_e32 v3, v2143; GFX942-NEXT: v_mov_b32_e32 v2, v1144; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v[2:3], s[0:3], s16 idxen offen sc0145; GFX942-NEXT: s_waitcnt vmcnt(0)146; GFX942-NEXT: s_setpc_b64 s[30:31]147;148; GFX1200-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:149; GFX1200: ; %bb.0:150; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0151; GFX1200-NEXT: s_wait_expcnt 0x0152; GFX1200-NEXT: s_wait_samplecnt 0x0153; GFX1200-NEXT: s_wait_bvhcnt 0x0154; GFX1200-NEXT: s_wait_kmcnt 0x0155; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN156; GFX1200-NEXT: s_wait_loadcnt 0x0157; GFX1200-NEXT: s_setpc_b64 s[30:31]158;159; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:160; GFX1250: ; %bb.0:161; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0162; GFX1250-NEXT: s_wait_kmcnt 0x0163; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1164; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN165; GFX1250-NEXT: s_wait_loadcnt 0x0166; GFX1250-NEXT: s_set_pc_i64 s[30:31]167 %ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)168 ret <2 x half> %ret169}170 171; Test waterfall loop172define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset) #0 {173; GFX90A-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:174; GFX90A: ; %bb.0:175; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)176; GFX90A-NEXT: v_mov_b32_e32 v9, v6177; GFX90A-NEXT: v_mov_b32_e32 v8, v5178; GFX90A-NEXT: v_mov_b32_e32 v5, v4179; GFX90A-NEXT: v_mov_b32_e32 v4, v3180; GFX90A-NEXT: v_mov_b32_e32 v3, v2181; GFX90A-NEXT: v_mov_b32_e32 v2, v1182; GFX90A-NEXT: s_mov_b64 s[6:7], exec183; GFX90A-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1184; GFX90A-NEXT: v_readfirstlane_b32 s8, v2185; GFX90A-NEXT: v_readfirstlane_b32 s9, v3186; GFX90A-NEXT: v_readfirstlane_b32 s10, v4187; GFX90A-NEXT: v_readfirstlane_b32 s11, v5188; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[2:3]189; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[4:5]190; GFX90A-NEXT: v_readfirstlane_b32 s12, v7191; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]192; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, s12, v7193; GFX90A-NEXT: s_and_b64 s[4:5], s[4:5], vcc194; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]195; GFX90A-NEXT: s_waitcnt vmcnt(0)196; GFX90A-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[8:11], s12 idxen offen glc197; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5198; GFX90A-NEXT: ; implicit-def: $vgpr7199; GFX90A-NEXT: ; implicit-def: $vgpr8_vgpr9200; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]201; GFX90A-NEXT: s_cbranch_execnz .LBB4_1202; GFX90A-NEXT: ; %bb.2:203; GFX90A-NEXT: s_mov_b64 exec, s[6:7]204; GFX90A-NEXT: s_waitcnt vmcnt(0)205; GFX90A-NEXT: s_setpc_b64 s[30:31]206;207; GFX942-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:208; GFX942: ; %bb.0:209; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)210; GFX942-NEXT: v_mov_b32_e32 v9, v6211; GFX942-NEXT: v_mov_b32_e32 v8, v5212; GFX942-NEXT: v_mov_b32_e32 v5, v4213; GFX942-NEXT: v_mov_b32_e32 v4, v3214; GFX942-NEXT: v_mov_b32_e32 v3, v2215; GFX942-NEXT: v_mov_b32_e32 v2, v1216; GFX942-NEXT: s_mov_b64 s[2:3], exec217; GFX942-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1218; GFX942-NEXT: v_readfirstlane_b32 s4, v2219; GFX942-NEXT: v_readfirstlane_b32 s5, v3220; GFX942-NEXT: v_readfirstlane_b32 s6, v4221; GFX942-NEXT: v_readfirstlane_b32 s7, v5222; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[2:3]223; GFX942-NEXT: v_readfirstlane_b32 s8, v7224; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[4:5]225; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]226; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, s8, v7227; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], vcc228; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]229; GFX942-NEXT: s_waitcnt vmcnt(0)230; GFX942-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[4:7], s8 idxen offen sc0231; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5232; GFX942-NEXT: ; implicit-def: $vgpr7233; GFX942-NEXT: ; implicit-def: $vgpr8_vgpr9234; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]235; GFX942-NEXT: s_cbranch_execnz .LBB4_1236; GFX942-NEXT: ; %bb.2:237; GFX942-NEXT: s_mov_b64 exec, s[2:3]238; GFX942-NEXT: s_waitcnt vmcnt(0)239; GFX942-NEXT: s_setpc_b64 s[30:31]240;241; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:242; GFX1200: ; %bb.0:243; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0244; GFX1200-NEXT: s_wait_expcnt 0x0245; GFX1200-NEXT: s_wait_samplecnt 0x0246; GFX1200-NEXT: s_wait_bvhcnt 0x0247; GFX1200-NEXT: s_wait_kmcnt 0x0248; GFX1200-NEXT: s_mov_b32 s2, exec_lo249; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1250; GFX1200-NEXT: v_readfirstlane_b32 s4, v1251; GFX1200-NEXT: v_readfirstlane_b32 s5, v2252; GFX1200-NEXT: v_readfirstlane_b32 s6, v3253; GFX1200-NEXT: v_readfirstlane_b32 s7, v4254; GFX1200-NEXT: v_readfirstlane_b32 s3, v7255; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)256; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]257; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)258; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]259; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7260; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0261; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)262; GFX1200-NEXT: s_and_b32 s0, s0, s1263; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)264; GFX1200-NEXT: s_and_saveexec_b32 s0, s0265; GFX1200-NEXT: s_wait_loadcnt 0x0266; GFX1200-NEXT: buffer_atomic_add_f32 v0, v[5:6], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN267; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4268; GFX1200-NEXT: ; implicit-def: $vgpr7269; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6270; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0271; GFX1200-NEXT: s_cbranch_execnz .LBB4_1272; GFX1200-NEXT: ; %bb.2:273; GFX1200-NEXT: s_mov_b32 exec_lo, s2274; GFX1200-NEXT: s_wait_loadcnt 0x0275; GFX1200-NEXT: s_setpc_b64 s[30:31]276;277; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:278; GFX1250: ; %bb.0:279; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0280; GFX1250-NEXT: s_wait_kmcnt 0x0281; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5282; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3283; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1284; GFX1250-NEXT: s_mov_b32 s2, exec_lo285; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1286; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)287; GFX1250-NEXT: v_readfirstlane_b32 s4, v2288; GFX1250-NEXT: v_readfirstlane_b32 s5, v3289; GFX1250-NEXT: v_readfirstlane_b32 s6, v4290; GFX1250-NEXT: v_readfirstlane_b32 s7, v5291; GFX1250-NEXT: v_readfirstlane_b32 s3, v7292; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)293; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]294; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]295; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)296; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7297; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0298; GFX1250-NEXT: s_and_b32 s0, s0, s1299; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)300; GFX1250-NEXT: s_and_saveexec_b32 s0, s0301; GFX1250-NEXT: s_wait_loadcnt 0x0302; GFX1250-NEXT: buffer_atomic_add_f32 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN303; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5304; GFX1250-NEXT: ; implicit-def: $vgpr7305; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9306; GFX1250-NEXT: s_wait_xcnt 0x0307; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0308; GFX1250-NEXT: s_cbranch_execnz .LBB4_1309; GFX1250-NEXT: ; %bb.2:310; GFX1250-NEXT: s_mov_b32 exec_lo, s2311; GFX1250-NEXT: s_wait_loadcnt 0x0312; GFX1250-NEXT: s_set_pc_i64 s[30:31]313 %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)314 ret float %ret315}316 317; Test waterfall loop318define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset) #0 {319; GFX90A-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:320; GFX90A: ; %bb.0:321; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)322; GFX90A-NEXT: v_mov_b32_e32 v9, v6323; GFX90A-NEXT: v_mov_b32_e32 v8, v5324; GFX90A-NEXT: v_mov_b32_e32 v5, v4325; GFX90A-NEXT: v_mov_b32_e32 v4, v3326; GFX90A-NEXT: v_mov_b32_e32 v3, v2327; GFX90A-NEXT: v_mov_b32_e32 v2, v1328; GFX90A-NEXT: s_mov_b64 s[6:7], exec329; GFX90A-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1330; GFX90A-NEXT: v_readfirstlane_b32 s8, v2331; GFX90A-NEXT: v_readfirstlane_b32 s9, v3332; GFX90A-NEXT: v_readfirstlane_b32 s10, v4333; GFX90A-NEXT: v_readfirstlane_b32 s11, v5334; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[2:3]335; GFX90A-NEXT: v_cmp_eq_u64_e64 s[4:5], s[10:11], v[4:5]336; GFX90A-NEXT: v_readfirstlane_b32 s12, v7337; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]338; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, s12, v7339; GFX90A-NEXT: s_and_b64 s[4:5], s[4:5], vcc340; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]341; GFX90A-NEXT: s_waitcnt vmcnt(0)342; GFX90A-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[8:11], s12 idxen offen glc343; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5344; GFX90A-NEXT: ; implicit-def: $vgpr7345; GFX90A-NEXT: ; implicit-def: $vgpr8_vgpr9346; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]347; GFX90A-NEXT: s_cbranch_execnz .LBB5_1348; GFX90A-NEXT: ; %bb.2:349; GFX90A-NEXT: s_mov_b64 exec, s[6:7]350; GFX90A-NEXT: s_waitcnt vmcnt(0)351; GFX90A-NEXT: s_setpc_b64 s[30:31]352;353; GFX942-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:354; GFX942: ; %bb.0:355; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)356; GFX942-NEXT: v_mov_b32_e32 v9, v6357; GFX942-NEXT: v_mov_b32_e32 v8, v5358; GFX942-NEXT: v_mov_b32_e32 v5, v4359; GFX942-NEXT: v_mov_b32_e32 v4, v3360; GFX942-NEXT: v_mov_b32_e32 v3, v2361; GFX942-NEXT: v_mov_b32_e32 v2, v1362; GFX942-NEXT: s_mov_b64 s[2:3], exec363; GFX942-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1364; GFX942-NEXT: v_readfirstlane_b32 s4, v2365; GFX942-NEXT: v_readfirstlane_b32 s5, v3366; GFX942-NEXT: v_readfirstlane_b32 s6, v4367; GFX942-NEXT: v_readfirstlane_b32 s7, v5368; GFX942-NEXT: v_cmp_eq_u64_e32 vcc, s[4:5], v[2:3]369; GFX942-NEXT: v_readfirstlane_b32 s8, v7370; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[4:5]371; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]372; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, s8, v7373; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], vcc374; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]375; GFX942-NEXT: s_waitcnt vmcnt(0)376; GFX942-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s8 idxen offen sc0377; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5378; GFX942-NEXT: ; implicit-def: $vgpr7379; GFX942-NEXT: ; implicit-def: $vgpr8_vgpr9380; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]381; GFX942-NEXT: s_cbranch_execnz .LBB5_1382; GFX942-NEXT: ; %bb.2:383; GFX942-NEXT: s_mov_b64 exec, s[2:3]384; GFX942-NEXT: s_waitcnt vmcnt(0)385; GFX942-NEXT: s_setpc_b64 s[30:31]386;387; GFX1200-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:388; GFX1200: ; %bb.0:389; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0390; GFX1200-NEXT: s_wait_expcnt 0x0391; GFX1200-NEXT: s_wait_samplecnt 0x0392; GFX1200-NEXT: s_wait_bvhcnt 0x0393; GFX1200-NEXT: s_wait_kmcnt 0x0394; GFX1200-NEXT: s_mov_b32 s2, exec_lo395; GFX1200-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1396; GFX1200-NEXT: v_readfirstlane_b32 s4, v1397; GFX1200-NEXT: v_readfirstlane_b32 s5, v2398; GFX1200-NEXT: v_readfirstlane_b32 s6, v3399; GFX1200-NEXT: v_readfirstlane_b32 s7, v4400; GFX1200-NEXT: v_readfirstlane_b32 s3, v7401; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)402; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]403; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)404; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]405; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v7406; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0407; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)408; GFX1200-NEXT: s_and_b32 s0, s0, s1409; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)410; GFX1200-NEXT: s_and_saveexec_b32 s0, s0411; GFX1200-NEXT: s_wait_loadcnt 0x0412; GFX1200-NEXT: buffer_atomic_pk_add_f16 v0, v[5:6], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN413; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4414; GFX1200-NEXT: ; implicit-def: $vgpr7415; GFX1200-NEXT: ; implicit-def: $vgpr5_vgpr6416; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0417; GFX1200-NEXT: s_cbranch_execnz .LBB5_1418; GFX1200-NEXT: ; %bb.2:419; GFX1200-NEXT: s_mov_b32 exec_lo, s2420; GFX1200-NEXT: s_wait_loadcnt 0x0421; GFX1200-NEXT: s_setpc_b64 s[30:31]422;423; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:424; GFX1250: ; %bb.0:425; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0426; GFX1250-NEXT: s_wait_kmcnt 0x0427; GFX1250-NEXT: v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5428; GFX1250-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3429; GFX1250-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1430; GFX1250-NEXT: s_mov_b32 s2, exec_lo431; GFX1250-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1432; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)433; GFX1250-NEXT: v_readfirstlane_b32 s4, v2434; GFX1250-NEXT: v_readfirstlane_b32 s5, v3435; GFX1250-NEXT: v_readfirstlane_b32 s6, v4436; GFX1250-NEXT: v_readfirstlane_b32 s7, v5437; GFX1250-NEXT: v_readfirstlane_b32 s3, v7438; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)439; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]440; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]441; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)442; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v7443; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0444; GFX1250-NEXT: s_and_b32 s0, s0, s1445; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)446; GFX1250-NEXT: s_and_saveexec_b32 s0, s0447; GFX1250-NEXT: s_wait_loadcnt 0x0448; GFX1250-NEXT: buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN449; GFX1250-NEXT: ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5450; GFX1250-NEXT: ; implicit-def: $vgpr7451; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9452; GFX1250-NEXT: s_wait_xcnt 0x0453; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0454; GFX1250-NEXT: s_cbranch_execnz .LBB5_1455; GFX1250-NEXT: ; %bb.2:456; GFX1250-NEXT: s_mov_b32 exec_lo, s2457; GFX1250-NEXT: s_wait_loadcnt 0x0458; GFX1250-NEXT: s_set_pc_i64 s[30:31]459 %ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)460 ret <2 x half> %ret461}462 463declare float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float, ptr addrspace(8), i32, i32, i32, i32 immarg)464declare <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half>, ptr addrspace(8), i32, i32, i32, i32 immarg)465 466attributes #0 = { nounwind }467