442 lines · cpp
1//===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// Implements the AMDGPU specific subclass of TargetSubtarget.11//12//===----------------------------------------------------------------------===//13 14#include "AMDGPUSubtarget.h"15#include "AMDGPUCallLowering.h"16#include "AMDGPUInstructionSelector.h"17#include "AMDGPULegalizerInfo.h"18#include "AMDGPURegisterBankInfo.h"19#include "R600Subtarget.h"20#include "SIMachineFunctionInfo.h"21#include "Utils/AMDGPUBaseInfo.h"22#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"23#include "llvm/CodeGen/MachineScheduler.h"24#include "llvm/CodeGen/TargetFrameLowering.h"25#include "llvm/IR/DiagnosticInfo.h"26#include "llvm/IR/IntrinsicsAMDGPU.h"27#include "llvm/IR/IntrinsicsR600.h"28#include "llvm/IR/MDBuilder.h"29#include <algorithm>30 31using namespace llvm;32 33#define DEBUG_TYPE "amdgpu-subtarget"34 35AMDGPUSubtarget::AMDGPUSubtarget(Triple TT) : TargetTriple(std::move(TT)) {}36 37bool AMDGPUSubtarget::useRealTrue16Insts() const {38 return hasTrue16BitInsts() && EnableRealTrue16Insts;39}40 41bool AMDGPUSubtarget::hasD16Writes32BitVgpr() const {42 return EnableD16Writes32BitVgpr;43}44 45// Returns the maximum per-workgroup LDS allocation size (in bytes) that still46// allows the given function to achieve an occupancy of NWaves waves per47// SIMD / EU, taking into account only the function's *maximum* workgroup size.48unsigned49AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,50 const Function &F) const {51 const unsigned WaveSize = getWavefrontSize();52 const unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;53 const unsigned WavesPerWorkgroup =54 std::max(1u, (WorkGroupSize + WaveSize - 1) / WaveSize);55 56 const unsigned WorkGroupsPerCU =57 std::max(1u, (NWaves * getEUsPerCU()) / WavesPerWorkgroup);58 59 return getLocalMemorySize() / WorkGroupsPerCU;60}61 62std::pair<unsigned, unsigned> AMDGPUSubtarget::getOccupancyWithWorkGroupSizes(63 uint32_t LDSBytes, std::pair<unsigned, unsigned> FlatWorkGroupSizes) const {64 65 // FIXME: We should take into account the LDS allocation granularity.66 const unsigned MaxWGsLDS = getLocalMemorySize() / std::max(LDSBytes, 1u);67 68 // Queried LDS size may be larger than available on a CU, in which case we69 // consider the only achievable occupancy to be 1, in line with what we70 // consider the occupancy to be when the number of requested registers in a71 // particular bank is higher than the number of available ones in that bank.72 if (!MaxWGsLDS)73 return {1, 1};74 75 const unsigned WaveSize = getWavefrontSize(), WavesPerEU = getMaxWavesPerEU();76 77 auto PropsFromWGSize = [=](unsigned WGSize)78 -> std::tuple<const unsigned, const unsigned, unsigned> {79 unsigned WavesPerWG = divideCeil(WGSize, WaveSize);80 unsigned WGsPerCU = std::min(getMaxWorkGroupsPerCU(WGSize), MaxWGsLDS);81 return {WavesPerWG, WGsPerCU, WavesPerWG * WGsPerCU};82 };83 84 // The maximum group size will generally yield the minimum number of85 // workgroups, maximum number of waves, and minimum occupancy. The opposite is86 // generally true for the minimum group size. LDS or barrier ressource87 // limitations can flip those minimums/maximums.88 const auto [MinWGSize, MaxWGSize] = FlatWorkGroupSizes;89 auto [MinWavesPerWG, MaxWGsPerCU, MaxWavesPerCU] = PropsFromWGSize(MinWGSize);90 auto [MaxWavesPerWG, MinWGsPerCU, MinWavesPerCU] = PropsFromWGSize(MaxWGSize);91 92 // It is possible that we end up with flipped minimum and maximum number of93 // waves per CU when the number of minimum/maximum concurrent groups on the CU94 // is limited by LDS usage or barrier resources.95 if (MinWavesPerCU >= MaxWavesPerCU) {96 std::swap(MinWavesPerCU, MaxWavesPerCU);97 } else {98 const unsigned WaveSlotsPerCU = WavesPerEU * getEUsPerCU();99 100 // Look for a potential smaller group size than the maximum which decreases101 // the concurrent number of waves on the CU for the same number of102 // concurrent workgroups on the CU.103 unsigned MinWavesPerCUForWGSize =104 divideCeil(WaveSlotsPerCU, MinWGsPerCU + 1) * MinWGsPerCU;105 if (MinWavesPerCU > MinWavesPerCUForWGSize) {106 unsigned ExcessSlots = MinWavesPerCU - MinWavesPerCUForWGSize;107 if (unsigned ExcessSlotsPerWG = ExcessSlots / MinWGsPerCU) {108 // There may exist a smaller group size than the maximum that achieves109 // the minimum number of waves per CU. This group size is the largest110 // possible size that requires MaxWavesPerWG - E waves where E is111 // maximized under the following constraints.112 // 1. 0 <= E <= ExcessSlotsPerWG113 // 2. (MaxWavesPerWG - E) * WaveSize >= MinWGSize114 MinWavesPerCU -= MinWGsPerCU * std::min(ExcessSlotsPerWG,115 MaxWavesPerWG - MinWavesPerWG);116 }117 }118 119 // Look for a potential larger group size than the minimum which increases120 // the concurrent number of waves on the CU for the same number of121 // concurrent workgroups on the CU.122 unsigned LeftoverSlots = WaveSlotsPerCU - MaxWGsPerCU * MinWavesPerWG;123 if (unsigned LeftoverSlotsPerWG = LeftoverSlots / MaxWGsPerCU) {124 // There may exist a larger group size than the minimum that achieves the125 // maximum number of waves per CU. This group size is the smallest126 // possible size that requires MinWavesPerWG + L waves where L is127 // maximized under the following constraints.128 // 1. 0 <= L <= LeftoverSlotsPerWG129 // 2. (MinWavesPerWG + L - 1) * WaveSize <= MaxWGSize130 MaxWavesPerCU += MaxWGsPerCU * std::min(LeftoverSlotsPerWG,131 ((MaxWGSize - 1) / WaveSize) + 1 -132 MinWavesPerWG);133 }134 }135 136 // Return the minimum/maximum number of waves on any EU, assuming that all137 // wavefronts are spread across all EUs as evenly as possible.138 return {std::clamp(MinWavesPerCU / getEUsPerCU(), 1U, WavesPerEU),139 std::clamp(divideCeil(MaxWavesPerCU, getEUsPerCU()), 1U, WavesPerEU)};140}141 142std::pair<unsigned, unsigned> AMDGPUSubtarget::getOccupancyWithWorkGroupSizes(143 const MachineFunction &MF) const {144 const auto *MFI = MF.getInfo<SIMachineFunctionInfo>();145 return getOccupancyWithWorkGroupSizes(MFI->getLDSSize(), MF.getFunction());146}147 148std::pair<unsigned, unsigned>149AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const {150 switch (CC) {151 case CallingConv::AMDGPU_VS:152 case CallingConv::AMDGPU_LS:153 case CallingConv::AMDGPU_HS:154 case CallingConv::AMDGPU_ES:155 case CallingConv::AMDGPU_GS:156 case CallingConv::AMDGPU_PS:157 return std::pair(1, getWavefrontSize());158 default:159 return std::pair(1u, getMaxFlatWorkGroupSize());160 }161}162 163std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(164 const Function &F) const {165 // Default minimum/maximum flat work group sizes.166 std::pair<unsigned, unsigned> Default =167 getDefaultFlatWorkGroupSize(F.getCallingConv());168 169 // Requested minimum/maximum flat work group sizes.170 std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(171 F, "amdgpu-flat-work-group-size", Default);172 173 // Make sure requested minimum is less than requested maximum.174 if (Requested.first > Requested.second)175 return Default;176 177 // Make sure requested values do not violate subtarget's specifications.178 if (Requested.first < getMinFlatWorkGroupSize())179 return Default;180 if (Requested.second > getMaxFlatWorkGroupSize())181 return Default;182 183 return Requested;184}185 186std::pair<unsigned, unsigned> AMDGPUSubtarget::getEffectiveWavesPerEU(187 std::pair<unsigned, unsigned> RequestedWavesPerEU,188 std::pair<unsigned, unsigned> FlatWorkGroupSizes, unsigned LDSBytes) const {189 // Default minimum/maximum number of waves per EU. The range of flat workgroup190 // sizes limits the achievable maximum, and we aim to support enough waves per191 // EU so that we can concurrently execute all waves of a single workgroup of192 // maximum size on a CU.193 std::pair<unsigned, unsigned> Default = {194 getWavesPerEUForWorkGroup(FlatWorkGroupSizes.second),195 getOccupancyWithWorkGroupSizes(LDSBytes, FlatWorkGroupSizes).second};196 Default.first = std::min(Default.first, Default.second);197 198 // Make sure requested minimum is within the default range and lower than the199 // requested maximum. The latter must not violate target specification.200 if (RequestedWavesPerEU.first < Default.first ||201 RequestedWavesPerEU.first > Default.second ||202 RequestedWavesPerEU.first > RequestedWavesPerEU.second ||203 RequestedWavesPerEU.second > getMaxWavesPerEU())204 return Default;205 206 // We cannot exceed maximum occupancy implied by flat workgroup size and LDS.207 RequestedWavesPerEU.second =208 std::min(RequestedWavesPerEU.second, Default.second);209 return RequestedWavesPerEU;210}211 212std::pair<unsigned, unsigned>213AMDGPUSubtarget::getWavesPerEU(const Function &F) const {214 // Default/requested minimum/maximum flat work group sizes.215 std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);216 // Minimum number of bytes allocated in the LDS.217 unsigned LDSBytes =218 AMDGPU::getIntegerPairAttribute(F, "amdgpu-lds-size", {0, UINT32_MAX},219 /*OnlyFirstRequired=*/true)220 .first;221 return getWavesPerEU(FlatWorkGroupSizes, LDSBytes, F);222}223 224std::pair<unsigned, unsigned>225AMDGPUSubtarget::getWavesPerEU(std::pair<unsigned, unsigned> FlatWorkGroupSizes,226 unsigned LDSBytes, const Function &F) const {227 // Default minimum/maximum number of waves per execution unit.228 std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());229 230 // Requested minimum/maximum number of waves per execution unit.231 std::pair<unsigned, unsigned> Requested =232 AMDGPU::getIntegerPairAttribute(F, "amdgpu-waves-per-eu", Default, true);233 return getEffectiveWavesPerEU(Requested, FlatWorkGroupSizes, LDSBytes);234}235 236std::optional<unsigned>237AMDGPUSubtarget::getReqdWorkGroupSize(const Function &Kernel,238 unsigned Dim) const {239 auto *Node = Kernel.getMetadata("reqd_work_group_size");240 if (Node && Node->getNumOperands() == 3)241 return mdconst::extract<ConstantInt>(Node->getOperand(Dim))->getZExtValue();242 return std::nullopt;243}244 245bool AMDGPUSubtarget::hasWavefrontsEvenlySplittingXDim(246 const Function &F, bool RequiresUniformYZ) const {247 auto *Node = F.getMetadata("reqd_work_group_size");248 if (!Node || Node->getNumOperands() != 3)249 return false;250 unsigned XLen =251 mdconst::extract<ConstantInt>(Node->getOperand(0))->getZExtValue();252 unsigned YLen =253 mdconst::extract<ConstantInt>(Node->getOperand(1))->getZExtValue();254 unsigned ZLen =255 mdconst::extract<ConstantInt>(Node->getOperand(2))->getZExtValue();256 257 bool Is1D = YLen <= 1 && ZLen <= 1;258 bool IsXLargeEnough =259 isPowerOf2_32(XLen) && (!RequiresUniformYZ || XLen >= getWavefrontSize());260 return Is1D || IsXLargeEnough;261}262 263bool AMDGPUSubtarget::isMesaKernel(const Function &F) const {264 return isMesa3DOS() && !AMDGPU::isShader(F.getCallingConv());265}266 267unsigned AMDGPUSubtarget::getMaxWorkitemID(const Function &Kernel,268 unsigned Dimension) const {269 std::optional<unsigned> ReqdSize = getReqdWorkGroupSize(Kernel, Dimension);270 if (ReqdSize)271 return *ReqdSize - 1;272 return getFlatWorkGroupSizes(Kernel).second - 1;273}274 275bool AMDGPUSubtarget::isSingleLaneExecution(const Function &Func) const {276 for (int I = 0; I < 3; ++I) {277 if (getMaxWorkitemID(Func, I) > 0)278 return false;279 }280 281 return true;282}283 284bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {285 Function *Kernel = I->getFunction();286 unsigned MinSize = 0;287 unsigned MaxSize = getFlatWorkGroupSizes(*Kernel).second;288 bool IdQuery = false;289 290 // If reqd_work_group_size is present it narrows value down.291 if (auto *CI = dyn_cast<CallInst>(I)) {292 const Function *F = CI->getCalledFunction();293 if (F) {294 unsigned Dim = UINT_MAX;295 switch (F->getIntrinsicID()) {296 case Intrinsic::amdgcn_workitem_id_x:297 case Intrinsic::r600_read_tidig_x:298 IdQuery = true;299 [[fallthrough]];300 case Intrinsic::r600_read_local_size_x:301 Dim = 0;302 break;303 case Intrinsic::amdgcn_workitem_id_y:304 case Intrinsic::r600_read_tidig_y:305 IdQuery = true;306 [[fallthrough]];307 case Intrinsic::r600_read_local_size_y:308 Dim = 1;309 break;310 case Intrinsic::amdgcn_workitem_id_z:311 case Intrinsic::r600_read_tidig_z:312 IdQuery = true;313 [[fallthrough]];314 case Intrinsic::r600_read_local_size_z:315 Dim = 2;316 break;317 default:318 break;319 }320 321 if (Dim <= 3) {322 std::optional<unsigned> ReqdSize = getReqdWorkGroupSize(*Kernel, Dim);323 if (ReqdSize)324 MinSize = MaxSize = *ReqdSize;325 }326 }327 }328 329 if (!MaxSize)330 return false;331 332 // Range metadata is [Lo, Hi). For ID query we need to pass max size333 // as Hi. For size query we need to pass Hi + 1.334 if (IdQuery)335 MinSize = 0;336 else337 ++MaxSize;338 339 APInt Lower{32, MinSize};340 APInt Upper{32, MaxSize};341 if (auto *CI = dyn_cast<CallBase>(I)) {342 ConstantRange Range(Lower, Upper);343 CI->addRangeRetAttr(Range);344 } else {345 MDBuilder MDB(I->getContext());346 MDNode *MaxWorkGroupSizeRange = MDB.createRange(Lower, Upper);347 I->setMetadata(LLVMContext::MD_range, MaxWorkGroupSizeRange);348 }349 return true;350}351 352unsigned AMDGPUSubtarget::getImplicitArgNumBytes(const Function &F) const {353 assert(AMDGPU::isKernel(F));354 355 // We don't allocate the segment if we know the implicit arguments weren't356 // used, even if the ABI implies we need them.357 if (F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))358 return 0;359 360 if (isMesaKernel(F))361 return 16;362 363 // Assume all implicit inputs are used by default364 const Module *M = F.getParent();365 unsigned NBytes =366 AMDGPU::getAMDHSACodeObjectVersion(*M) >= AMDGPU::AMDHSA_COV5 ? 256 : 56;367 return F.getFnAttributeAsParsedInteger("amdgpu-implicitarg-num-bytes",368 NBytes);369}370 371uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,372 Align &MaxAlign) const {373 assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||374 F.getCallingConv() == CallingConv::SPIR_KERNEL);375 376 const DataLayout &DL = F.getDataLayout();377 uint64_t ExplicitArgBytes = 0;378 MaxAlign = Align(1);379 380 for (const Argument &Arg : F.args()) {381 if (Arg.hasAttribute("amdgpu-hidden-argument"))382 continue;383 384 const bool IsByRef = Arg.hasByRefAttr();385 Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();386 Align Alignment = DL.getValueOrABITypeAlignment(387 IsByRef ? Arg.getParamAlign() : std::nullopt, ArgTy);388 uint64_t AllocSize = DL.getTypeAllocSize(ArgTy);389 ExplicitArgBytes = alignTo(ExplicitArgBytes, Alignment) + AllocSize;390 MaxAlign = std::max(MaxAlign, Alignment);391 }392 393 return ExplicitArgBytes;394}395 396unsigned AMDGPUSubtarget::getKernArgSegmentSize(const Function &F,397 Align &MaxAlign) const {398 if (F.getCallingConv() != CallingConv::AMDGPU_KERNEL &&399 F.getCallingConv() != CallingConv::SPIR_KERNEL)400 return 0;401 402 uint64_t ExplicitArgBytes = getExplicitKernArgSize(F, MaxAlign);403 404 unsigned ExplicitOffset = getExplicitKernelArgOffset();405 406 uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes;407 unsigned ImplicitBytes = getImplicitArgNumBytes(F);408 if (ImplicitBytes != 0) {409 const Align Alignment = getAlignmentForImplicitArgPtr();410 TotalSize = alignTo(ExplicitArgBytes, Alignment) + ImplicitBytes;411 MaxAlign = std::max(MaxAlign, Alignment);412 }413 414 // Being able to dereference past the end is useful for emitting scalar loads.415 return alignTo(TotalSize, 4);416}417 418AMDGPUDwarfFlavour AMDGPUSubtarget::getAMDGPUDwarfFlavour() const {419 return getWavefrontSize() == 32 ? AMDGPUDwarfFlavour::Wave32420 : AMDGPUDwarfFlavour::Wave64;421}422 423const AMDGPUSubtarget &AMDGPUSubtarget::get(const MachineFunction &MF) {424 if (MF.getTarget().getTargetTriple().isAMDGCN())425 return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<GCNSubtarget>());426 return static_cast<const AMDGPUSubtarget &>(MF.getSubtarget<R600Subtarget>());427}428 429const AMDGPUSubtarget &AMDGPUSubtarget::get(const TargetMachine &TM, const Function &F) {430 if (TM.getTargetTriple().isAMDGCN())431 return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<GCNSubtarget>(F));432 return static_cast<const AMDGPUSubtarget &>(433 TM.getSubtarget<R600Subtarget>(F));434}435 436// FIXME: This has no reason to be in subtarget437SmallVector<unsigned>438AMDGPUSubtarget::getMaxNumWorkGroups(const Function &F) const {439 return AMDGPU::getIntegerVecAttribute(F, "amdgpu-max-num-workgroups", 3,440 std::numeric_limits<uint32_t>::max());441}442