274 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx942 | FileCheck %s -check-prefix=GFX9423; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 | FileCheck %s -check-prefix=GFX124; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 | FileCheck %s -check-prefix=GFX12505 6declare <2 x half> @llvm.amdgcn.ds.fadd.v2f16(ptr addrspace(3) %ptr, <2 x half> %data, i32, i32, i1)7declare <2 x i16> @llvm.amdgcn.ds.fadd.v2bf16(ptr addrspace(3) %ptr, <2 x i16> %data)8 9define amdgpu_kernel void @flat_atomic_fadd_f32_noret_pat(ptr %ptr) {10; GFX942-LABEL: flat_atomic_fadd_f32_noret_pat:11; GFX942: ; %bb.0:12; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2413; GFX942-NEXT: v_mov_b32_e32 v2, 4.014; GFX942-NEXT: s_waitcnt lgkmcnt(0)15; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]16; GFX942-NEXT: buffer_wbl2 sc0 sc117; GFX942-NEXT: flat_atomic_add_f32 v[0:1], v2 sc118; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)19; GFX942-NEXT: buffer_inv sc0 sc120; GFX942-NEXT: s_endpgm21;22; GFX12-LABEL: flat_atomic_fadd_f32_noret_pat:23; GFX12: ; %bb.0:24; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x2425; GFX12-NEXT: v_mov_b32_e32 v2, 4.026; GFX12-NEXT: s_wait_kmcnt 0x027; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s128; GFX12-NEXT: global_wb scope:SCOPE_SYS29; GFX12-NEXT: s_wait_storecnt 0x030; GFX12-NEXT: flat_atomic_add_f32 v[0:1], v2 scope:SCOPE_SYS31; GFX12-NEXT: s_wait_storecnt_dscnt 0x032; GFX12-NEXT: global_inv scope:SCOPE_SYS33; GFX12-NEXT: s_endpgm34;35; GFX1250-LABEL: flat_atomic_fadd_f32_noret_pat:36; GFX1250: ; %bb.0:37; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 138; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x2439; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 4.040; GFX1250-NEXT: global_wb scope:SCOPE_SYS41; GFX1250-NEXT: s_wait_storecnt 0x042; GFX1250-NEXT: s_wait_kmcnt 0x043; GFX1250-NEXT: flat_atomic_add_f32 v0, v1, s[0:1] scope:SCOPE_SYS44; GFX1250-NEXT: s_wait_storecnt_dscnt 0x045; GFX1250-NEXT: global_inv scope:SCOPE_SYS46; GFX1250-NEXT: s_endpgm47 %ret = atomicrmw fadd ptr %ptr, float 4.0 seq_cst, !amdgpu.no.remote.memory !048 ret void49}50 51define amdgpu_kernel void @flat_atomic_fadd_f32_noret_pat_ieee(ptr %ptr) #0 {52; GFX942-LABEL: flat_atomic_fadd_f32_noret_pat_ieee:53; GFX942: ; %bb.0:54; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2455; GFX942-NEXT: v_mov_b32_e32 v2, 4.056; GFX942-NEXT: s_waitcnt lgkmcnt(0)57; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[0:1]58; GFX942-NEXT: buffer_wbl2 sc0 sc159; GFX942-NEXT: flat_atomic_add_f32 v[0:1], v2 sc160; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)61; GFX942-NEXT: buffer_inv sc0 sc162; GFX942-NEXT: s_endpgm63;64; GFX12-LABEL: flat_atomic_fadd_f32_noret_pat_ieee:65; GFX12: ; %bb.0:66; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x2467; GFX12-NEXT: v_mov_b32_e32 v2, 4.068; GFX12-NEXT: s_wait_kmcnt 0x069; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s170; GFX12-NEXT: global_wb scope:SCOPE_SYS71; GFX12-NEXT: s_wait_storecnt 0x072; GFX12-NEXT: flat_atomic_add_f32 v[0:1], v2 scope:SCOPE_SYS73; GFX12-NEXT: s_wait_storecnt_dscnt 0x074; GFX12-NEXT: global_inv scope:SCOPE_SYS75; GFX12-NEXT: s_endpgm76;77; GFX1250-LABEL: flat_atomic_fadd_f32_noret_pat_ieee:78; GFX1250: ; %bb.0:79; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 180; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x2481; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 4.082; GFX1250-NEXT: global_wb scope:SCOPE_SYS83; GFX1250-NEXT: s_wait_storecnt 0x084; GFX1250-NEXT: s_wait_kmcnt 0x085; GFX1250-NEXT: flat_atomic_add_f32 v0, v1, s[0:1] scope:SCOPE_SYS86; GFX1250-NEXT: s_wait_storecnt_dscnt 0x087; GFX1250-NEXT: global_inv scope:SCOPE_SYS88; GFX1250-NEXT: s_endpgm89 %ret = atomicrmw fadd ptr %ptr, float 4.0 seq_cst, !amdgpu.no.remote.memory !090 ret void91}92define float @flat_atomic_fadd_f32_rtn_pat(ptr %ptr, float %data) {93; GFX942-LABEL: flat_atomic_fadd_f32_rtn_pat:94; GFX942: ; %bb.0:95; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)96; GFX942-NEXT: v_mov_b32_e32 v2, 4.097; GFX942-NEXT: buffer_wbl2 sc0 sc198; GFX942-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 sc0 sc199; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)100; GFX942-NEXT: buffer_inv sc0 sc1101; GFX942-NEXT: s_setpc_b64 s[30:31]102;103; GFX12-LABEL: flat_atomic_fadd_f32_rtn_pat:104; GFX12: ; %bb.0:105; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0106; GFX12-NEXT: s_wait_expcnt 0x0107; GFX12-NEXT: s_wait_samplecnt 0x0108; GFX12-NEXT: s_wait_bvhcnt 0x0109; GFX12-NEXT: s_wait_kmcnt 0x0110; GFX12-NEXT: v_mov_b32_e32 v2, 4.0111; GFX12-NEXT: global_wb scope:SCOPE_SYS112; GFX12-NEXT: s_wait_storecnt 0x0113; GFX12-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS114; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0115; GFX12-NEXT: global_inv scope:SCOPE_SYS116; GFX12-NEXT: s_wait_loadcnt 0x0117; GFX12-NEXT: s_setpc_b64 s[30:31]118;119; GFX1250-LABEL: flat_atomic_fadd_f32_rtn_pat:120; GFX1250: ; %bb.0:121; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0122; GFX1250-NEXT: s_wait_kmcnt 0x0123; GFX1250-NEXT: v_mov_b32_e32 v2, 4.0124; GFX1250-NEXT: global_wb scope:SCOPE_SYS125; GFX1250-NEXT: s_wait_storecnt 0x0126; GFX1250-NEXT: flat_atomic_add_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS127; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0128; GFX1250-NEXT: global_inv scope:SCOPE_SYS129; GFX1250-NEXT: s_wait_loadcnt 0x0130; GFX1250-NEXT: s_set_pc_i64 s[30:31]131 %ret = atomicrmw fadd ptr %ptr, float 4.0 seq_cst, !amdgpu.no.remote.memory !0132 ret float %ret133}134 135define amdgpu_kernel void @local_atomic_fadd_v2f16_noret(ptr addrspace(3) %ptr, <2 x half> %data) {136; GFX942-LABEL: local_atomic_fadd_v2f16_noret:137; GFX942: ; %bb.0:138; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24139; GFX942-NEXT: s_waitcnt lgkmcnt(0)140; GFX942-NEXT: v_mov_b32_e32 v0, s0141; GFX942-NEXT: v_mov_b32_e32 v1, s1142; GFX942-NEXT: ds_pk_add_f16 v0, v1143; GFX942-NEXT: s_waitcnt lgkmcnt(0)144; GFX942-NEXT: s_endpgm145;146; GFX12-LABEL: local_atomic_fadd_v2f16_noret:147; GFX12: ; %bb.0:148; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24149; GFX12-NEXT: s_wait_kmcnt 0x0150; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1151; GFX12-NEXT: ds_pk_add_f16 v0, v1152; GFX12-NEXT: s_wait_dscnt 0x0153; GFX12-NEXT: global_inv scope:SCOPE_SE154; GFX12-NEXT: s_endpgm155;156; GFX1250-LABEL: local_atomic_fadd_v2f16_noret:157; GFX1250: ; %bb.0:158; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1159; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24160; GFX1250-NEXT: s_wait_kmcnt 0x0161; GFX1250-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1162; GFX1250-NEXT: ds_pk_add_f16 v0, v1163; GFX1250-NEXT: s_wait_dscnt 0x0164; GFX1250-NEXT: s_endpgm165 %ret = call <2 x half> @llvm.amdgcn.ds.fadd.v2f16(ptr addrspace(3) %ptr, <2 x half> %data, i32 0, i32 0, i1 0)166 ret void167}168 169define <2 x half> @local_atomic_fadd_v2f16_rtn(ptr addrspace(3) %ptr, <2 x half> %data) {170; GFX942-LABEL: local_atomic_fadd_v2f16_rtn:171; GFX942: ; %bb.0:172; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)173; GFX942-NEXT: ds_pk_add_rtn_f16 v0, v0, v1174; GFX942-NEXT: s_waitcnt lgkmcnt(0)175; GFX942-NEXT: s_setpc_b64 s[30:31]176;177; GFX12-LABEL: local_atomic_fadd_v2f16_rtn:178; GFX12: ; %bb.0:179; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0180; GFX12-NEXT: s_wait_expcnt 0x0181; GFX12-NEXT: s_wait_samplecnt 0x0182; GFX12-NEXT: s_wait_bvhcnt 0x0183; GFX12-NEXT: s_wait_kmcnt 0x0184; GFX12-NEXT: s_wait_storecnt 0x0185; GFX12-NEXT: ds_pk_add_rtn_f16 v0, v0, v1186; GFX12-NEXT: s_wait_dscnt 0x0187; GFX12-NEXT: global_inv scope:SCOPE_SE188; GFX12-NEXT: s_wait_loadcnt 0x0189; GFX12-NEXT: s_setpc_b64 s[30:31]190;191; GFX1250-LABEL: local_atomic_fadd_v2f16_rtn:192; GFX1250: ; %bb.0:193; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0194; GFX1250-NEXT: s_wait_kmcnt 0x0195; GFX1250-NEXT: s_wait_storecnt 0x0196; GFX1250-NEXT: ds_pk_add_rtn_f16 v0, v0, v1197; GFX1250-NEXT: s_wait_dscnt 0x0198; GFX1250-NEXT: s_set_pc_i64 s[30:31]199 %ret = call <2 x half> @llvm.amdgcn.ds.fadd.v2f16(ptr addrspace(3) %ptr, <2 x half> %data, i32 0, i32 0, i1 0)200 ret <2 x half> %ret201}202 203define amdgpu_kernel void @local_atomic_fadd_v2bf16_noret(ptr addrspace(3) %ptr, <2 x i16> %data) {204; GFX942-LABEL: local_atomic_fadd_v2bf16_noret:205; GFX942: ; %bb.0:206; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24207; GFX942-NEXT: s_waitcnt lgkmcnt(0)208; GFX942-NEXT: v_mov_b32_e32 v0, s0209; GFX942-NEXT: v_mov_b32_e32 v1, s1210; GFX942-NEXT: ds_pk_add_bf16 v0, v1211; GFX942-NEXT: s_waitcnt lgkmcnt(0)212; GFX942-NEXT: s_endpgm213;214; GFX12-LABEL: local_atomic_fadd_v2bf16_noret:215; GFX12: ; %bb.0:216; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24217; GFX12-NEXT: s_wait_kmcnt 0x0218; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1219; GFX12-NEXT: ds_pk_add_bf16 v0, v1220; GFX12-NEXT: s_wait_dscnt 0x0221; GFX12-NEXT: global_inv scope:SCOPE_SE222; GFX12-NEXT: s_endpgm223;224; GFX1250-LABEL: local_atomic_fadd_v2bf16_noret:225; GFX1250: ; %bb.0:226; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1227; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24228; GFX1250-NEXT: s_wait_kmcnt 0x0229; GFX1250-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1230; GFX1250-NEXT: ds_pk_add_bf16 v0, v1231; GFX1250-NEXT: s_wait_dscnt 0x0232; GFX1250-NEXT: s_endpgm233 %ret = call <2 x i16> @llvm.amdgcn.ds.fadd.v2bf16(ptr addrspace(3) %ptr, <2 x i16> %data)234 ret void235}236 237define <2 x i16> @local_atomic_fadd_v2bf16_rtn(ptr addrspace(3) %ptr, <2 x i16> %data) {238; GFX942-LABEL: local_atomic_fadd_v2bf16_rtn:239; GFX942: ; %bb.0:240; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)241; GFX942-NEXT: ds_pk_add_rtn_bf16 v0, v0, v1242; GFX942-NEXT: s_waitcnt lgkmcnt(0)243; GFX942-NEXT: s_setpc_b64 s[30:31]244;245; GFX12-LABEL: local_atomic_fadd_v2bf16_rtn:246; GFX12: ; %bb.0:247; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0248; GFX12-NEXT: s_wait_expcnt 0x0249; GFX12-NEXT: s_wait_samplecnt 0x0250; GFX12-NEXT: s_wait_bvhcnt 0x0251; GFX12-NEXT: s_wait_kmcnt 0x0252; GFX12-NEXT: s_wait_storecnt 0x0253; GFX12-NEXT: ds_pk_add_rtn_bf16 v0, v0, v1254; GFX12-NEXT: s_wait_dscnt 0x0255; GFX12-NEXT: global_inv scope:SCOPE_SE256; GFX12-NEXT: s_wait_loadcnt 0x0257; GFX12-NEXT: s_setpc_b64 s[30:31]258;259; GFX1250-LABEL: local_atomic_fadd_v2bf16_rtn:260; GFX1250: ; %bb.0:261; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0262; GFX1250-NEXT: s_wait_kmcnt 0x0263; GFX1250-NEXT: s_wait_storecnt 0x0264; GFX1250-NEXT: ds_pk_add_rtn_bf16 v0, v0, v1265; GFX1250-NEXT: s_wait_dscnt 0x0266; GFX1250-NEXT: s_set_pc_i64 s[30:31]267 %ret = call <2 x i16> @llvm.amdgcn.ds.fadd.v2bf16(ptr addrspace(3) %ptr, <2 x i16> %data)268 ret <2 x i16> %ret269}270 271attributes #0 = { "denormal-fp-math-f32"="ieee,ieee" }272 273!0 = !{}274