236 lines · plain
1; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,LOOP %s2; RUN: llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti < %s | FileCheck -check-prefixes=GCN,LOOP %s3; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii < %s | FileCheck -check-prefixes=GCN,LOOP %s4; RUN: llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii < %s | FileCheck -check-prefixes=GCN,LOOP %s5; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,LOOP %s6; RUN: llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,LOOP %s7; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,NOLOOP,GFX9 %s8; RUN: llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,NOLOOP,GFX9 %s9; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -asm-verbose=0 < %s | FileCheck -check-prefixes=GCN,NOLOOP,GFX10 %s10; RUN: llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -asm-verbose=0 < %s | FileCheck -check-prefixes=GCN,NOLOOP,GFX10 %s11; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -asm-verbose=0 < %s | FileCheck -check-prefixes=GCN,NOLOOP,GFX10 %s12; RUN: llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -asm-verbose=0 < %s | FileCheck -check-prefixes=GCN,NOLOOP,GFX10 %s13 14; Make sure the op is emitted bundled with a waitcnt with and without the retry loop, and the bundle is not removed by ExpandPostRAPseudos.15; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -stop-after=postrapseudos < %s | FileCheck -check-prefix=MIR %s16; RUN: llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -stop-after=postrapseudos < %s | FileCheck -check-prefix=MIR %s17; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -stop-after=postrapseudos < %s | FileCheck -check-prefix=MIR %s18; RUN: llc -global-isel=1 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -stop-after=postrapseudos < %s | FileCheck -check-prefix=MIR %s19 20 21; Minimum offset22; GCN-LABEL: {{^}}gws_barrier_offset0:23; NOLOOP-DAG: s_load_{{dword|b32}} [[BAR_NUM:s[0-9]+]]24; NOLOOP-DAG: s_mov_b32 m0, 0{{$}}25; NOLOOP: v_mov_b32_e32 v0, [[BAR_NUM]]26; NOLOOP: ds_gws_barrier v0 gds{{$}}27 28; LOOP: s_mov_b32 m0, 0{{$}}29; LOOP: [[LOOP:.LBB[0-9]+_[0-9]+]]:30; LOOP-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_TRAPSTS, 8, 1), 031; LOOP-NEXT: ds_gws_barrier v0 gds32; LOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)33; LOOP-NEXT: s_getreg_b32 [[GETREG:s[0-9]+]], hwreg(HW_REG_TRAPSTS, 8, 1)34; LOOP-NEXT: s_cmp_lg_u32 [[GETREG]], 035; LOOP-NEXT: s_cbranch_scc1 [[LOOP]]36 37; MIR-LABEL: name: gws_barrier_offset0{{$}}38; MIR: BUNDLE implicit{{( killed)?( renamable)?}} $vgpr0, implicit $m0, implicit $exec39; MIR-NEXT: DS_GWS_BARRIER renamable $vgpr0, 0, implicit $m0, implicit $exec :: (load (s32) from custom "GWSResource")40; MIR-NEXT: S_WAITCNT 041; MIR-NEXT: }42define amdgpu_kernel void @gws_barrier_offset0(i32 %val) #0 {43 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 0)44 ret void45}46 47; MIR-LABEL: name: gws_barrier_offset63{{$}}48 49; Maximum offset50; GCN-LABEL: {{^}}gws_barrier_offset63:51; NOLOOP-DAG: s_load_{{dword|b32}} [[BAR_NUM:s[0-9]+]]52; NOLOOP-DAG: s_mov_b32 m0, 0{{$}}53; NOLOOP-DAG: v_mov_b32_e32 v0, [[BAR_NUM]]54; NOLOOP: ds_gws_barrier v0 offset:63 gds{{$}}55define amdgpu_kernel void @gws_barrier_offset63(i32 %val) #0 {56 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 63)57 ret void58}59 60; FIXME: Should be able to shift directly into m061; GCN-LABEL: {{^}}gws_barrier_sgpr_offset:62; NOLOOP-DAG: s_load_{{dwordx2|b64}} s[[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]]63 64; NOLOOP-DAG: s_lshl_b32 m0, s[[OFFSET]], 1665 66; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]]67; NOLOOP: ds_gws_barrier [[GWS_VAL]] gds{{$}}68define amdgpu_kernel void @gws_barrier_sgpr_offset(i32 %val, i32 %offset) #0 {69 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %offset)70 ret void71}72 73; Variable offset in SGPR with constant add74; GCN-LABEL: {{^}}gws_barrier_sgpr_offset_add1:75; NOLOOP-DAG: s_load_{{dwordx2|b64}} s[[[BAR_NUM:[0-9]+]]:[[OFFSET:[0-9]+]]]76 77; NOLOOP-DAG: s_lshl_b32 m0, s[[OFFSET]], 1678 79; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], s[[BAR_NUM]]80; NOLOOP: ds_gws_barrier [[GWS_VAL]] offset:1 gds{{$}}81define amdgpu_kernel void @gws_barrier_sgpr_offset_add1(i32 %val, i32 %offset.base) #0 {82 %offset = add i32 %offset.base, 183 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %offset)84 ret void85}86 87; GCN-LABEL: {{^}}gws_barrier_vgpr_offset:88; NOLOOP-DAG: s_load_{{dword|b32}} [[BAR_NUM:s[0-9]+]]89; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v090 91; NOLOOP-DAG: s_lshl_b32 m0, [[READLANE]], 1692 93; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], [[BAR_NUM]]94; NOLOOP: ds_gws_barrier [[GWS_VAL]] gds{{$}}95define amdgpu_kernel void @gws_barrier_vgpr_offset(i32 %val) #0 {96 %vgpr.offset = call i32 @llvm.amdgcn.workitem.id.x()97 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %vgpr.offset)98 ret void99}100 101; Variable offset in VGPR with constant add102; GCN-LABEL: {{^}}gws_barrier_vgpr_offset_add:103; NOLOOP-DAG: s_load_{{dword|b32}} [[BAR_NUM:s[0-9]+]]104; NOLOOP-DAG: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0105 106; NOLOOP-DAG: s_lshl_b32 m0, [[READLANE]], 16107 108; NOLOOP-DAG: v_mov_b32_e32 [[GWS_VAL:v[0-9]+]], [[BAR_NUM]]109; NOLOOP: ds_gws_barrier [[GWS_VAL]] offset:3 gds{{$}}110define amdgpu_kernel void @gws_barrier_vgpr_offset_add(i32 %val) #0 {111 %vgpr.offset.base = call i32 @llvm.amdgcn.workitem.id.x()112 %vgpr.offset = add i32 %vgpr.offset.base, 3113 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 %vgpr.offset)114 ret void115}116 117@lds = internal unnamed_addr addrspace(3) global i32 poison118 119; Check if m0 initialization is shared120; GCN-LABEL: {{^}}gws_barrier_save_m0_barrier_constant_offset:121; NOLOOP: s_mov_b32 m0, 0122; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:10 gds123 124; LOOP: s_mov_b32 m0, -1125; LOOP: ds_write_b32126; LOOP: s_mov_b32 m0, 0127; LOOP: s_setreg_imm32_b32128; LOOP: ds_gws_barrier v{{[0-9]+}} offset:10 gds129; LOOP: s_cbranch_scc1130 131; LOOP: s_mov_b32 m0, -1132; LOOP: ds_write_b32133define amdgpu_kernel void @gws_barrier_save_m0_barrier_constant_offset(i32 %val) #0 {134 store i32 1, ptr addrspace(3) @lds135 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 10)136 store i32 2, ptr addrspace(3) @lds137 ret void138}139 140; Make sure this increments lgkmcnt141; GCN-LABEL: {{^}}gws_barrier_lgkmcnt:142; NOLOOP: s_mov_b32 m0, 0{{$}}143; NOLOOP: ds_gws_barrier v0 gds{{$}}144; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)145; NOLOOP-NEXT: s_setpc_b64146define void @gws_barrier_lgkmcnt(i32 %val) {147 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 0)148 ret void149}150 151; Does not imply memory fence on its own152; GCN-LABEL: {{^}}gws_barrier_wait_before:153; NOLOOP: s_waitcnt154; NOLOOP-NOT: s_waitcnt{{$}}155define amdgpu_kernel void @gws_barrier_wait_before(i32 %val, ptr addrspace(1) %ptr) #0 {156 store i32 0, ptr addrspace(1) %ptr157 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)158 ret void159}160 161; GCN-LABEL: {{^}}gws_barrier_wait_after:162; NOLOOP: s_mov_b32 m0, 0{{$}}163; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:7 gds164; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)165; NOLOOP: load_{{dword|b32}}166define amdgpu_kernel void @gws_barrier_wait_after(i32 %val, ptr addrspace(1) %ptr) #0 {167 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)168 %load = load volatile i32, ptr addrspace(1) %ptr169 ret void170}171 172; Does not imply memory fence on its own173; GCN-LABEL: {{^}}gws_barrier_fence_before:174; NOLOOP: s_mov_b32 m0, 0{{$}}175; NOLOOP: store_{{dword|b32}}176; GFX9: s_waitcnt vmcnt(0)177; GFX10: s_waitcnt_vscnt null, 0x0178; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:7 gds179; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)180define amdgpu_kernel void @gws_barrier_fence_before(i32 %val, ptr addrspace(1) %ptr) #0 {181 store i32 0, ptr addrspace(1) %ptr182 fence release183 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)184 ret void185}186 187; FIXME: Extra waitcnt188; GCN-LABEL: {{^}}gws_barrier_fence_after:189; NOLOOP: s_mov_b32 m0, 0{{$}}190; NOLOOP: ds_gws_barrier v{{[0-9]+}} offset:7 gds191; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)192; NOLOOP-NEXT: load_{{dword|b32}}193 194define amdgpu_kernel void @gws_barrier_fence_after(i32 %val, ptr addrspace(1) %ptr) #0 {195 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)196 fence release197 %load = load volatile i32, ptr addrspace(1) %ptr198 ret void199}200 201; FIXME: Should a wait be inserted here, or is an explicit fence needed?202; GCN-LABEL: {{^}}gws_init_barrier:203; NOLOOP: s_mov_b32 m0, 0204; NOLOOP: ds_gws_init v0 offset:7 gds205; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)206; NOLOOP-NEXT: ds_gws_barrier v0 offset:7 gds207; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)208define amdgpu_kernel void @gws_init_barrier(i32 %val) #0 {209 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7)210 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)211 ret void212}213 214; FIXME: Why vmcnt, not expcnt?215; GCN-LABEL: {{^}}gws_init_fence_barrier:216; NOLOOP: s_mov_b32 m0, 0217; NOLOOP: ds_gws_init v0 offset:7 gds218; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)219; NOLOOP-NEXT: ds_gws_barrier v0 offset:7 gds220; NOLOOP-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)221define amdgpu_kernel void @gws_init_fence_barrier(i32 %val) #0 {222 call void @llvm.amdgcn.ds.gws.init(i32 %val, i32 7)223 fence release224 call void @llvm.amdgcn.ds.gws.barrier(i32 %val, i32 7)225 ret void226}227 228declare void @llvm.amdgcn.ds.gws.barrier(i32, i32) #1229declare void @llvm.amdgcn.ds.gws.init(i32, i32) #2230declare i32 @llvm.amdgcn.workitem.id.x() #3231 232attributes #0 = { nounwind }233attributes #1 = { convergent inaccessiblememonly nounwind }234attributes #2 = { convergent inaccessiblememonly nounwind writeonly }235attributes #3 = { nounwind readnone speculatable }236