180 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; FIXME: Test 90a, 940. 908 should fail to select.3; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s4; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s5 6define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset(<2 x bfloat> %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) #0 {7; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:8; GFX1200: ; %bb.0:9; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x010; GFX1200-NEXT: s_wait_expcnt 0x011; GFX1200-NEXT: s_wait_samplecnt 0x012; GFX1200-NEXT: s_wait_bvhcnt 0x013; GFX1200-NEXT: s_wait_kmcnt 0x014; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s16 offen offset:128 th:TH_ATOMIC_RETURN15; GFX1200-NEXT: s_wait_loadcnt 0x016; GFX1200-NEXT: s_setpc_b64 s[30:31]17;18; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:19; GFX1250: ; %bb.0:20; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x021; GFX1250-NEXT: s_wait_kmcnt 0x022; GFX1250-NEXT: v_add_nc_u32_e32 v1, 0x80, v123; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_RETURN24; GFX1250-NEXT: s_wait_loadcnt 0x025; GFX1250-NEXT: s_set_pc_i64 s[30:31]26 %voffset.add = add i32 %voffset, 12827 %ret = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %voffset.add, i32 %soffset, i32 0)28 ret <2 x bfloat> %ret29}30 31define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc(<2 x bfloat> %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) #0 {32; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc:33; GFX1200: ; %bb.0:34; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x035; GFX1200-NEXT: s_wait_expcnt 0x036; GFX1200-NEXT: s_wait_samplecnt 0x037; GFX1200-NEXT: s_wait_bvhcnt 0x038; GFX1200-NEXT: s_wait_kmcnt 0x039; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_NT_RETURN40; GFX1200-NEXT: s_wait_loadcnt 0x041; GFX1200-NEXT: s_setpc_b64 s[30:31]42;43; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc:44; GFX1250: ; %bb.0:45; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x046; GFX1250-NEXT: s_wait_kmcnt 0x047; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_NT_RETURN48; GFX1250-NEXT: s_wait_loadcnt 0x049; GFX1250-NEXT: s_set_pc_i64 s[30:31]50 %ret = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 92, i32 %soffset, i32 2)51 ret <2 x bfloat> %ret52}53 54define void @raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset(<2 x bfloat> %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) #0 {55; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:56; GFX1200: ; %bb.0:57; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x058; GFX1200-NEXT: s_wait_expcnt 0x059; GFX1200-NEXT: s_wait_samplecnt 0x060; GFX1200-NEXT: s_wait_bvhcnt 0x061; GFX1200-NEXT: s_wait_kmcnt 0x062; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s16 offen offset:12863; GFX1200-NEXT: s_setpc_b64 s[30:31]64;65; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:66; GFX1250: ; %bb.0:67; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x068; GFX1250-NEXT: s_wait_kmcnt 0x069; GFX1250-NEXT: v_add_nc_u32_e32 v1, 0x80, v170; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s16 offen71; GFX1250-NEXT: s_set_pc_i64 s[30:31]72 %voffset.add = add i32 %voffset, 12873 %unused = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %voffset.add, i32 %soffset, i32 0)74 ret void75}76 77define void @raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc(<2 x bfloat> %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) #0 {78; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc:79; GFX1200: ; %bb.0:80; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x081; GFX1200-NEXT: s_wait_expcnt 0x082; GFX1200-NEXT: s_wait_samplecnt 0x083; GFX1200-NEXT: s_wait_bvhcnt 0x084; GFX1200-NEXT: s_wait_kmcnt 0x085; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_NT86; GFX1200-NEXT: s_setpc_b64 s[30:31]87;88; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc:89; GFX1250: ; %bb.0:90; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x091; GFX1250-NEXT: s_wait_kmcnt 0x092; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_NT93; GFX1250-NEXT: s_set_pc_i64 s[30:31]94 %unused = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 92, i32 %soffset, i32 2)95 ret void96}97 98; Test waterfall loop99define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset_add__vgpr_soffset(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset) #0 {100; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset_add__vgpr_soffset:101; GFX1200: ; %bb.0:102; GFX1200-NEXT: s_wait_loadcnt_dscnt 0x0103; GFX1200-NEXT: s_wait_expcnt 0x0104; GFX1200-NEXT: s_wait_samplecnt 0x0105; GFX1200-NEXT: s_wait_bvhcnt 0x0106; GFX1200-NEXT: s_wait_kmcnt 0x0107; GFX1200-NEXT: s_mov_b32 s2, exec_lo108; GFX1200-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1109; GFX1200-NEXT: v_readfirstlane_b32 s4, v1110; GFX1200-NEXT: v_readfirstlane_b32 s5, v2111; GFX1200-NEXT: v_readfirstlane_b32 s6, v3112; GFX1200-NEXT: v_readfirstlane_b32 s7, v4113; GFX1200-NEXT: v_readfirstlane_b32 s3, v6114; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)115; GFX1200-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]116; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)117; GFX1200-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]118; GFX1200-NEXT: v_cmp_eq_u32_e64 s1, s3, v6119; GFX1200-NEXT: s_and_b32 s0, vcc_lo, s0120; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)121; GFX1200-NEXT: s_and_b32 s0, s0, s1122; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)123; GFX1200-NEXT: s_and_saveexec_b32 s0, s0124; GFX1200-NEXT: s_wait_loadcnt 0x0125; GFX1200-NEXT: buffer_atomic_pk_add_bf16 v0, v5, s[4:7], s3 offen offset:128 th:TH_ATOMIC_RETURN126; GFX1200-NEXT: ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4127; GFX1200-NEXT: ; implicit-def: $vgpr6128; GFX1200-NEXT: ; implicit-def: $vgpr5129; GFX1200-NEXT: s_xor_b32 exec_lo, exec_lo, s0130; GFX1200-NEXT: s_cbranch_execnz .LBB4_1131; GFX1200-NEXT: ; %bb.2:132; GFX1200-NEXT: s_mov_b32 exec_lo, s2133; GFX1200-NEXT: s_wait_loadcnt 0x0134; GFX1200-NEXT: s_setpc_b64 s[30:31]135;136; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset_add__vgpr_soffset:137; GFX1250: ; %bb.0:138; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0139; GFX1250-NEXT: s_wait_kmcnt 0x0140; GFX1250-NEXT: v_dual_mov_b32 v11, v4 :: v_dual_mov_b32 v10, v3141; GFX1250-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v8, v1142; GFX1250-NEXT: v_add_nc_u32_e32 v1, 0x80, v5143; GFX1250-NEXT: s_mov_b32 s2, exec_lo144; GFX1250-NEXT: .LBB4_1: ; =>This Inner Loop Header: Depth=1145; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)146; GFX1250-NEXT: v_readfirstlane_b32 s4, v8147; GFX1250-NEXT: v_readfirstlane_b32 s5, v9148; GFX1250-NEXT: v_readfirstlane_b32 s6, v10149; GFX1250-NEXT: v_readfirstlane_b32 s7, v11150; GFX1250-NEXT: v_readfirstlane_b32 s3, v6151; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)152; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[8:9]153; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[10:11]154; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)155; GFX1250-NEXT: v_cmp_eq_u32_e64 s1, s3, v6156; GFX1250-NEXT: s_and_b32 s0, vcc_lo, s0157; GFX1250-NEXT: s_and_b32 s0, s0, s1158; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)159; GFX1250-NEXT: s_and_saveexec_b32 s0, s0160; GFX1250-NEXT: s_wait_loadcnt 0x0161; GFX1250-NEXT: buffer_atomic_pk_add_bf16 v0, v1, s[4:7], s3 offen th:TH_ATOMIC_RETURN162; GFX1250-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11163; GFX1250-NEXT: ; implicit-def: $vgpr6164; GFX1250-NEXT: ; implicit-def: $vgpr1165; GFX1250-NEXT: s_wait_xcnt 0x0166; GFX1250-NEXT: s_xor_b32 exec_lo, exec_lo, s0167; GFX1250-NEXT: s_cbranch_execnz .LBB4_1168; GFX1250-NEXT: ; %bb.2:169; GFX1250-NEXT: s_mov_b32 exec_lo, s2170; GFX1250-NEXT: s_wait_loadcnt 0x0171; GFX1250-NEXT: s_set_pc_i64 s[30:31]172 %voffset.add = add i32 %voffset, 128173 %ret = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %voffset.add, i32 %soffset, i32 0)174 ret <2 x bfloat> %ret175}176 177declare <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat>, ptr addrspace(8), i32, i32, i32 immarg)178 179attributes #0 = { nounwind }180