242 lines · cpp
1//===-- AMDGPUMachineFunctionInfo.cpp ---------------------------------------=//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "AMDGPUMachineFunction.h"10#include "AMDGPU.h"11#include "AMDGPUMemoryUtils.h"12#include "AMDGPUSubtarget.h"13#include "Utils/AMDGPUBaseInfo.h"14#include "llvm/CodeGen/MachineModuleInfo.h"15#include "llvm/IR/ConstantRange.h"16#include "llvm/IR/Constants.h"17#include "llvm/IR/Metadata.h"18#include "llvm/Target/TargetMachine.h"19 20using namespace llvm;21 22static const GlobalVariable *23getKernelDynLDSGlobalFromFunction(const Function &F) {24 const Module *M = F.getParent();25 SmallString<64> KernelDynLDSName("llvm.amdgcn.");26 KernelDynLDSName += F.getName();27 KernelDynLDSName += ".dynlds";28 return M->getNamedGlobal(KernelDynLDSName);29}30 31static bool hasLDSKernelArgument(const Function &F) {32 for (const Argument &Arg : F.args()) {33 Type *ArgTy = Arg.getType();34 if (auto *PtrTy = dyn_cast<PointerType>(ArgTy)) {35 if (PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)36 return true;37 }38 }39 return false;40}41 42AMDGPUMachineFunction::AMDGPUMachineFunction(const Function &F,43 const AMDGPUSubtarget &ST)44 : IsEntryFunction(AMDGPU::isEntryFunctionCC(F.getCallingConv())),45 IsModuleEntryFunction(46 AMDGPU::isModuleEntryFunctionCC(F.getCallingConv())),47 IsChainFunction(AMDGPU::isChainCC(F.getCallingConv())) {48 49 // FIXME: Should initialize KernArgSize based on ExplicitKernelArgOffset,50 // except reserved size is not correctly aligned.51 52 Attribute MemBoundAttr = F.getFnAttribute("amdgpu-memory-bound");53 MemoryBound = MemBoundAttr.getValueAsBool();54 55 Attribute WaveLimitAttr = F.getFnAttribute("amdgpu-wave-limiter");56 WaveLimiter = WaveLimitAttr.getValueAsBool();57 58 // FIXME: How is this attribute supposed to interact with statically known59 // global sizes?60 StringRef S = F.getFnAttribute("amdgpu-gds-size").getValueAsString();61 if (!S.empty())62 S.consumeInteger(0, GDSSize);63 64 // Assume the attribute allocates before any known GDS globals.65 StaticGDSSize = GDSSize;66 67 // Second value, if present, is the maximum value that can be assigned.68 // Useful in PromoteAlloca or for LDS spills. Could be used for diagnostics69 // during codegen.70 std::pair<unsigned, unsigned> LDSSizeRange = AMDGPU::getIntegerPairAttribute(71 F, "amdgpu-lds-size", {0, UINT32_MAX}, true);72 73 // The two separate variables are only profitable when the LDS module lowering74 // pass is disabled. If graphics does not use dynamic LDS, this is never75 // profitable. Leaving cleanup for a later change.76 LDSSize = LDSSizeRange.first;77 StaticLDSSize = LDSSize;78 79 CallingConv::ID CC = F.getCallingConv();80 if (CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL)81 ExplicitKernArgSize = ST.getExplicitKernArgSize(F, MaxKernArgAlign);82 83 // FIXME: Shouldn't be target specific84 Attribute NSZAttr = F.getFnAttribute("no-signed-zeros-fp-math");85 NoSignedZerosFPMath =86 NSZAttr.isStringAttribute() && NSZAttr.getValueAsString() == "true";87 88 const GlobalVariable *DynLdsGlobal = getKernelDynLDSGlobalFromFunction(F);89 if (DynLdsGlobal || hasLDSKernelArgument(F))90 UsesDynamicLDS = true;91}92 93unsigned AMDGPUMachineFunction::allocateLDSGlobal(const DataLayout &DL,94 const GlobalVariable &GV,95 Align Trailing) {96 auto Entry = LocalMemoryObjects.insert(std::pair(&GV, 0));97 if (!Entry.second)98 return Entry.first->second;99 100 Align Alignment =101 DL.getValueOrABITypeAlignment(GV.getAlign(), GV.getValueType());102 103 unsigned Offset;104 if (GV.getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {105 if (AMDGPU::isNamedBarrier(GV)) {106 std::optional<unsigned> BarAddr = getLDSAbsoluteAddress(GV);107 if (!BarAddr)108 llvm_unreachable("named barrier should have an assigned address");109 Entry.first->second = BarAddr.value();110 unsigned BarCnt = DL.getTypeAllocSize(GV.getValueType()) / 16;111 recordNumNamedBarriers(BarAddr.value(), BarCnt);112 return BarAddr.value();113 }114 115 std::optional<uint32_t> MaybeAbs = getLDSAbsoluteAddress(GV);116 if (MaybeAbs) {117 // Absolute address LDS variables that exist prior to the LDS lowering118 // pass raise a fatal error in that pass. These failure modes are only119 // reachable if that lowering pass is disabled or broken. If/when adding120 // support for absolute addresses on user specified variables, the121 // alignment check moves to the lowering pass and the frame calculation122 // needs to take the user variables into consideration.123 124 uint32_t ObjectStart = *MaybeAbs;125 126 if (ObjectStart != alignTo(ObjectStart, Alignment)) {127 report_fatal_error("Absolute address LDS variable inconsistent with "128 "variable alignment");129 }130 131 if (isModuleEntryFunction()) {132 // If this is a module entry function, we can also sanity check against133 // the static frame. Strictly it would be better to check against the134 // attribute, i.e. that the variable is within the always-allocated135 // section, and not within some other non-absolute-address object136 // allocated here, but the extra error detection is minimal and we would137 // have to pass the Function around or cache the attribute value.138 uint32_t ObjectEnd =139 ObjectStart + DL.getTypeAllocSize(GV.getValueType());140 if (ObjectEnd > StaticLDSSize) {141 report_fatal_error(142 "Absolute address LDS variable outside of static frame");143 }144 }145 146 Entry.first->second = ObjectStart;147 return ObjectStart;148 }149 150 /// TODO: We should sort these to minimize wasted space due to alignment151 /// padding. Currently the padding is decided by the first encountered use152 /// during lowering.153 Offset = StaticLDSSize = alignTo(StaticLDSSize, Alignment);154 155 StaticLDSSize += DL.getTypeAllocSize(GV.getValueType());156 157 // Align LDS size to trailing, e.g. for aligning dynamic shared memory158 LDSSize = alignTo(StaticLDSSize, Trailing);159 } else {160 assert(GV.getAddressSpace() == AMDGPUAS::REGION_ADDRESS &&161 "expected region address space");162 163 Offset = StaticGDSSize = alignTo(StaticGDSSize, Alignment);164 StaticGDSSize += DL.getTypeAllocSize(GV.getValueType());165 166 // FIXME: Apply alignment of dynamic GDS167 GDSSize = StaticGDSSize;168 }169 170 Entry.first->second = Offset;171 return Offset;172}173 174std::optional<uint32_t>175AMDGPUMachineFunction::getLDSKernelIdMetadata(const Function &F) {176 // TODO: Would be more consistent with the abs symbols to use a range177 MDNode *MD = F.getMetadata("llvm.amdgcn.lds.kernel.id");178 if (MD && MD->getNumOperands() == 1) {179 if (ConstantInt *KnownSize =180 mdconst::extract<ConstantInt>(MD->getOperand(0))) {181 uint64_t ZExt = KnownSize->getZExtValue();182 if (ZExt <= UINT32_MAX) {183 return ZExt;184 }185 }186 }187 return {};188}189 190std::optional<uint32_t>191AMDGPUMachineFunction::getLDSAbsoluteAddress(const GlobalValue &GV) {192 if (GV.getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)193 return {};194 195 std::optional<ConstantRange> AbsSymRange = GV.getAbsoluteSymbolRange();196 if (!AbsSymRange)197 return {};198 199 if (const APInt *V = AbsSymRange->getSingleElement()) {200 std::optional<uint64_t> ZExt = V->tryZExtValue();201 if (ZExt && (*ZExt <= UINT32_MAX)) {202 return *ZExt;203 }204 }205 206 return {};207}208 209void AMDGPUMachineFunction::setDynLDSAlign(const Function &F,210 const GlobalVariable &GV) {211 const Module *M = F.getParent();212 const DataLayout &DL = M->getDataLayout();213 assert(DL.getTypeAllocSize(GV.getValueType()).isZero());214 215 Align Alignment =216 DL.getValueOrABITypeAlignment(GV.getAlign(), GV.getValueType());217 if (Alignment <= DynLDSAlign)218 return;219 220 LDSSize = alignTo(StaticLDSSize, Alignment);221 DynLDSAlign = Alignment;222 223 // If there is a dynamic LDS variable associated with this function F, every224 // further dynamic LDS instance (allocated by calling setDynLDSAlign) must225 // map to the same address. This holds because no LDS is allocated after the226 // lowering pass if there are dynamic LDS variables present.227 const GlobalVariable *Dyn = getKernelDynLDSGlobalFromFunction(F);228 if (Dyn) {229 unsigned Offset = LDSSize; // return this?230 std::optional<uint32_t> Expect = getLDSAbsoluteAddress(*Dyn);231 if (!Expect || (Offset != *Expect)) {232 report_fatal_error("Inconsistent metadata on dynamic LDS variable");233 }234 }235}236 237void AMDGPUMachineFunction::setUsesDynamicLDS(bool DynLDS) {238 UsesDynamicLDS = DynLDS;239}240 241bool AMDGPUMachineFunction::isDynamicLDSUsed() const { return UsesDynamicLDS; }242