178 lines · plain
1; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck %s --check-prefixes=GCN,GFX92; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 < %s | FileCheck %s --check-prefixes=GCN,GFX103 4@lds.0 = internal addrspace(3) global [64 x float] poison, align 165@lds.1 = internal addrspace(3) global [64 x float] poison, align 166@lds.2 = internal addrspace(3) global [64 x float] poison, align 167@lds.3 = internal addrspace(3) global [64 x float] poison, align 168@lds.4 = internal addrspace(3) global [64 x float] poison, align 169@lds.5 = internal addrspace(3) global [64 x float] poison, align 1610@lds.6 = internal addrspace(3) global [64 x float] poison, align 1611@lds.7 = internal addrspace(3) global [64 x float] poison, align 1612@lds.8 = internal addrspace(3) global [64 x float] poison, align 1613@lds.9 = internal addrspace(3) global [64 x float] poison, align 1614 15declare void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) nocapture, i32 %size, i32 %voffset, i32 %soffset, i32 %offset, i32 %aux)16declare void @llvm.amdgcn.global.load.lds(ptr addrspace(1) nocapture %gptr, ptr addrspace(3) nocapture %lptr, i32 %size, i32 %offset, i32 %aux)17 18; GCN-LABEL: {{^}}buffer_load_lds_dword_2_arrays:19; GCN-COUNT-4: buffer_load_dword20; GCN: s_waitcnt vmcnt(2)21; GCN: ds_read_b3222; GCN: s_waitcnt vmcnt(0)23; GCN: ds_read_b3224define amdgpu_kernel void @buffer_load_lds_dword_2_arrays(<4 x i32> %rsrc, i32 %i1, i32 %i2, ptr addrspace(1) %out) {25main_body:26 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.0, i32 4, i32 0, i32 0, i32 0, i32 0)27 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.0, i32 4, i32 4, i32 0, i32 0, i32 0)28 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.1, i32 4, i32 8, i32 0, i32 0, i32 0)29 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.1, i32 4, i32 12, i32 0, i32 0, i32 0)30 %gep.0 = getelementptr float, ptr addrspace(3) @lds.0, i32 %i131 %gep.1 = getelementptr float, ptr addrspace(3) @lds.1, i32 %i232 %val.0 = load float, ptr addrspace(3) %gep.0, align 433 call void @llvm.amdgcn.wave.barrier()34 %val.1 = load float, ptr addrspace(3) %gep.1, align 435 %tmp.0 = insertelement <2 x float> poison, float %val.0, i32 036 %res = insertelement <2 x float> %tmp.0, float %val.1, i32 137 store <2 x float> %res, ptr addrspace(1) %out38 ret void39}40 41; On gfx9 if there is a pending FLAT operation, and this is a VMem or LGKM42; waitcnt and the target can report early completion, then we need to force a waitcnt 0.43 44; GCN-LABEL: {{^}}global_load_lds_dword_2_arrays:45; GCN-COUNT-4: global_load_dword46; GFX9: s_waitcnt vmcnt(0)47; GFX9-COUNT-2: ds_read_b3248; GFX10: s_waitcnt vmcnt(2)49; GFX10: ds_read_b3250; GFX10: s_waitcnt vmcnt(0)51; GFX10: ds_read_b3252define amdgpu_kernel void @global_load_lds_dword_2_arrays(ptr addrspace(1) nocapture %gptr, i32 %i1, i32 %i2, ptr addrspace(1) %out) {53main_body:54 call void @llvm.amdgcn.global.load.lds(ptr addrspace(1) %gptr, ptr addrspace(3) @lds.0, i32 4, i32 0, i32 0)55 call void @llvm.amdgcn.global.load.lds(ptr addrspace(1) %gptr, ptr addrspace(3) @lds.0, i32 4, i32 4, i32 0)56 call void @llvm.amdgcn.global.load.lds(ptr addrspace(1) %gptr, ptr addrspace(3) @lds.1, i32 4, i32 8, i32 0)57 call void @llvm.amdgcn.global.load.lds(ptr addrspace(1) %gptr, ptr addrspace(3) @lds.1, i32 4, i32 12, i32 0)58 %gep.0 = getelementptr float, ptr addrspace(3) @lds.0, i32 %i159 %gep.1 = getelementptr float, ptr addrspace(3) @lds.1, i32 %i260 %val.0 = load float, ptr addrspace(3) %gep.0, align 461 call void @llvm.amdgcn.wave.barrier()62 %val.1 = load float, ptr addrspace(3) %gep.1, align 463 %tmp.0 = insertelement <2 x float> poison, float %val.0, i32 064 %res = insertelement <2 x float> %tmp.0, float %val.1, i32 165 store <2 x float> %res, ptr addrspace(1) %out66 ret void67}68 69; There are 8 pseudo registers defined to track LDS DMA dependencies.70 71; GCN-LABEL: {{^}}buffer_load_lds_dword_10_arrays:72; GCN-COUNT-10: buffer_load_dword73; GCN: s_waitcnt vmcnt(8)74; GCN: ds_read_b3275; GCN: s_waitcnt vmcnt(7)76; GCN: ds_read_b3277; GCN: s_waitcnt vmcnt(6)78; GCN: ds_read_b3279; GCN: s_waitcnt vmcnt(5)80; GCN: ds_read_b3281; GCN: s_waitcnt vmcnt(4)82; GCN: ds_read_b3283; GCN: s_waitcnt vmcnt(3)84; GCN: ds_read_b3285; GCN: s_waitcnt vmcnt(2)86; GCN-NOT: s_waitcnt vmcnt87; GCN: ds_read_b3288; GCN: ds_read_b3289define amdgpu_kernel void @buffer_load_lds_dword_10_arrays(<4 x i32> %rsrc, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6, i32 %i7, i32 %i8, i32 %i9, ptr addrspace(1) %out) {90main_body:91 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.0, i32 4, i32 0, i32 0, i32 0, i32 0)92 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.1, i32 4, i32 0, i32 0, i32 0, i32 0)93 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.2, i32 4, i32 0, i32 0, i32 0, i32 0)94 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.3, i32 4, i32 0, i32 0, i32 0, i32 0)95 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.4, i32 4, i32 0, i32 0, i32 0, i32 0)96 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.5, i32 4, i32 0, i32 0, i32 0, i32 0)97 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.6, i32 4, i32 0, i32 0, i32 0, i32 0)98 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.7, i32 4, i32 0, i32 0, i32 0, i32 0)99 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.8, i32 4, i32 0, i32 0, i32 0, i32 0)100 call void @llvm.amdgcn.raw.buffer.load.lds(<4 x i32> %rsrc, ptr addrspace(3) @lds.9, i32 4, i32 0, i32 0, i32 0, i32 0)101 %gep.0 = getelementptr float, ptr addrspace(3) @lds.0, i32 %i1102 %gep.1 = getelementptr float, ptr addrspace(3) @lds.1, i32 %i2103 %gep.2 = getelementptr float, ptr addrspace(3) @lds.2, i32 %i2104 %gep.3 = getelementptr float, ptr addrspace(3) @lds.3, i32 %i2105 %gep.4 = getelementptr float, ptr addrspace(3) @lds.4, i32 %i2106 %gep.5 = getelementptr float, ptr addrspace(3) @lds.5, i32 %i2107 %gep.6 = getelementptr float, ptr addrspace(3) @lds.6, i32 %i2108 %gep.7 = getelementptr float, ptr addrspace(3) @lds.7, i32 %i2109 %gep.8 = getelementptr float, ptr addrspace(3) @lds.8, i32 %i2110 %gep.9 = getelementptr float, ptr addrspace(3) @lds.9, i32 %i2111 %val.0 = load float, ptr addrspace(3) %gep.0, align 4112 call void @llvm.amdgcn.wave.barrier()113 %val.1 = load float, ptr addrspace(3) %gep.1, align 4114 call void @llvm.amdgcn.wave.barrier()115 %val.2 = load float, ptr addrspace(3) %gep.2, align 4116 call void @llvm.amdgcn.wave.barrier()117 %val.3 = load float, ptr addrspace(3) %gep.3, align 4118 call void @llvm.amdgcn.wave.barrier()119 %val.4 = load float, ptr addrspace(3) %gep.4, align 4120 call void @llvm.amdgcn.wave.barrier()121 %val.5 = load float, ptr addrspace(3) %gep.5, align 4122 call void @llvm.amdgcn.wave.barrier()123 %val.6 = load float, ptr addrspace(3) %gep.6, align 4124 call void @llvm.amdgcn.wave.barrier()125 %val.7 = load float, ptr addrspace(3) %gep.7, align 4126 call void @llvm.amdgcn.wave.barrier()127 %val.8 = load float, ptr addrspace(3) %gep.8, align 4128 call void @llvm.amdgcn.wave.barrier()129 %val.9 = load float, ptr addrspace(3) %gep.9, align 4130 %out.gep.1 = getelementptr float, ptr addrspace(1) %out, i32 1131 %out.gep.2 = getelementptr float, ptr addrspace(1) %out, i32 2132 %out.gep.3 = getelementptr float, ptr addrspace(1) %out, i32 3133 %out.gep.4 = getelementptr float, ptr addrspace(1) %out, i32 4134 %out.gep.5 = getelementptr float, ptr addrspace(1) %out, i32 5135 %out.gep.6 = getelementptr float, ptr addrspace(1) %out, i32 6136 %out.gep.7 = getelementptr float, ptr addrspace(1) %out, i32 7137 %out.gep.8 = getelementptr float, ptr addrspace(1) %out, i32 8138 %out.gep.9 = getelementptr float, ptr addrspace(1) %out, i32 9139 store float %val.0, ptr addrspace(1) %out140 store float %val.1, ptr addrspace(1) %out.gep.1141 store float %val.2, ptr addrspace(1) %out.gep.2142 store float %val.3, ptr addrspace(1) %out.gep.3143 store float %val.4, ptr addrspace(1) %out.gep.4144 store float %val.5, ptr addrspace(1) %out.gep.5145 store float %val.6, ptr addrspace(1) %out.gep.6146 store float %val.7, ptr addrspace(1) %out.gep.7147 store float %val.8, ptr addrspace(1) %out.gep.8148 store float %val.9, ptr addrspace(1) %out.gep.9149 ret void150}151 152define amdgpu_kernel void @global_load_lds_no_alias_ds_read(ptr addrspace(1) nocapture %gptr, i32 %i1, i32 %i2, ptr addrspace(1) %out) {153; GFX9-LABEL: global_load_lds_no_alias_ds_read:154; GFX9: global_load_dword155; GFX9: global_load_dword156; GFX9: s_waitcnt vmcnt(1)157; GFX9-NOT: s_waitcnt vmcnt(0)158; GFX9: ds_read_b32159; GFX9: s_waitcnt vmcnt(0)160; GFX9: ds_read_b32161; GFX9: s_endpgm162body:163 call void @llvm.amdgcn.global.load.lds(ptr addrspace(1) %gptr, ptr addrspace(3) @lds.0, i32 4, i32 0, i32 0)164 call void @llvm.amdgcn.global.load.lds(ptr addrspace(1) %gptr, ptr addrspace(3) @lds.1, i32 4, i32 4, i32 0)165 call void @llvm.amdgcn.s.waitcnt(i32 3953)166 %gep.0 = getelementptr float, ptr addrspace(3) @lds.2, i32 %i1167 %val.0 = load float, ptr addrspace(3) %gep.0, align 4168 call void @llvm.amdgcn.s.waitcnt(i32 3952)169 %gep.1 = getelementptr float, ptr addrspace(3) @lds.3, i32 %i2170 %val.1 = load float, ptr addrspace(3) %gep.1, align 4171 %tmp = insertelement <2 x float> poison, float %val.0, i32 0172 %res = insertelement <2 x float> %tmp, float %val.1, i32 1173 store <2 x float> %res, ptr addrspace(1) %out174 ret void175}176 177declare void @llvm.amdgcn.wave.barrier()178