845 lines · cpp
1//===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "SIMachineFunctionInfo.h"10#include "AMDGPUSubtarget.h"11#include "GCNSubtarget.h"12#include "MCTargetDesc/AMDGPUMCTargetDesc.h"13#include "SIRegisterInfo.h"14#include "Utils/AMDGPUBaseInfo.h"15#include "llvm/CodeGen/LiveIntervals.h"16#include "llvm/CodeGen/MIRParser/MIParser.h"17#include "llvm/CodeGen/MachineBasicBlock.h"18#include "llvm/CodeGen/MachineFrameInfo.h"19#include "llvm/CodeGen/MachineFunction.h"20#include "llvm/CodeGen/MachineRegisterInfo.h"21#include "llvm/IR/CallingConv.h"22#include "llvm/IR/DiagnosticInfo.h"23#include "llvm/IR/Function.h"24#include <cassert>25#include <optional>26#include <vector>27 28enum { MAX_LANES = 64 };29 30using namespace llvm;31 32// TODO -- delete this flag once we have more robust mechanisms to allocate the33// optimal RC for Opc and Dest of MFMA. In particular, there are high RP cases34// where it is better to produce the VGPR form (e.g. if there are VGPR users35// of the MFMA result).36static cl::opt<bool, true> MFMAVGPRFormOpt(37 "amdgpu-mfma-vgpr-form",38 cl::desc("Whether to force use VGPR for Opc and Dest of MFMA. If "39 "unspecified, default to compiler heuristics"),40 cl::location(SIMachineFunctionInfo::MFMAVGPRForm), cl::init(false),41 cl::Hidden);42 43const GCNTargetMachine &getTM(const GCNSubtarget *STI) {44 const SITargetLowering *TLI = STI->getTargetLowering();45 return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine());46}47 48bool SIMachineFunctionInfo::MFMAVGPRForm = false;49 50SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F,51 const GCNSubtarget *STI)52 : AMDGPUMachineFunction(F, *STI), Mode(F, *STI), GWSResourcePSV(getTM(STI)),53 UserSGPRInfo(F, *STI), WorkGroupIDX(false), WorkGroupIDY(false),54 WorkGroupIDZ(false), WorkGroupInfo(false), LDSKernelId(false),55 PrivateSegmentWaveByteOffset(false), WorkItemIDX(false),56 WorkItemIDY(false), WorkItemIDZ(false), ImplicitArgPtr(false),57 GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0),58 IsWholeWaveFunction(F.getCallingConv() ==59 CallingConv::AMDGPU_Gfx_WholeWave) {60 const GCNSubtarget &ST = *STI;61 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);62 WavesPerEU = ST.getWavesPerEU(F);63 MaxNumWorkGroups = ST.getMaxNumWorkGroups(F);64 assert(MaxNumWorkGroups.size() == 3);65 66 // Temporarily check both the attribute and the subtarget feature, until the67 // latter is completely removed.68 DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);69 if (DynamicVGPRBlockSize == 0 && ST.isDynamicVGPREnabled())70 DynamicVGPRBlockSize = ST.getDynamicVGPRBlockSize();71 72 Occupancy = ST.computeOccupancy(F, getLDSSize()).second;73 CallingConv::ID CC = F.getCallingConv();74 75 VRegFlags.reserve(1024);76 77 const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||78 CC == CallingConv::SPIR_KERNEL;79 80 if (IsKernel) {81 WorkGroupIDX = true;82 WorkItemIDX = true;83 } else if (CC == CallingConv::AMDGPU_PS) {84 PSInputAddr = AMDGPU::getInitialPSInputAddr(F);85 }86 87 if (ST.hasGFX90AInsts()) {88 // FIXME: Extract logic out of getMaxNumVectorRegs; we need to apply the89 // allocation granule and clamping.90 auto [MinNumAGPRAttr, MaxNumAGPRAttr] =91 AMDGPU::getIntegerPairAttribute(F, "amdgpu-agpr-alloc", {~0u, ~0u},92 /*OnlyFirstRequired=*/true);93 MinNumAGPRs = MinNumAGPRAttr;94 }95 96 if (AMDGPU::isChainCC(CC)) {97 // Chain functions don't receive an SP from their caller, but are free to98 // set one up. For now, we can use s32 to match what amdgpu_gfx functions99 // would use if called, but this can be revisited.100 // FIXME: Only reserve this if we actually need it.101 StackPtrOffsetReg = AMDGPU::SGPR32;102 103 ScratchRSrcReg = AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51;104 105 ArgInfo.PrivateSegmentBuffer =106 ArgDescriptor::createRegister(ScratchRSrcReg);107 108 ImplicitArgPtr = false;109 } else if (!isEntryFunction()) {110 if (CC != CallingConv::AMDGPU_Gfx &&111 CC != CallingConv::AMDGPU_Gfx_WholeWave)112 ArgInfo = AMDGPUArgumentUsageInfo::FixedABIFunctionInfo;113 114 FrameOffsetReg = AMDGPU::SGPR33;115 StackPtrOffsetReg = AMDGPU::SGPR32;116 117 if (!ST.enableFlatScratch()) {118 // Non-entry functions have no special inputs for now, other registers119 // required for scratch access.120 ScratchRSrcReg = AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;121 122 ArgInfo.PrivateSegmentBuffer =123 ArgDescriptor::createRegister(ScratchRSrcReg);124 }125 126 if (!F.hasFnAttribute("amdgpu-no-implicitarg-ptr"))127 ImplicitArgPtr = true;128 } else {129 ImplicitArgPtr = false;130 MaxKernArgAlign =131 std::max(ST.getAlignmentForImplicitArgPtr(), MaxKernArgAlign);132 }133 134 if (!AMDGPU::isGraphics(CC) ||135 ((CC == CallingConv::AMDGPU_CS || CC == CallingConv::AMDGPU_Gfx) &&136 ST.hasArchitectedSGPRs())) {137 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workgroup-id-x") ||138 !F.hasFnAttribute("amdgpu-no-cluster-id-x"))139 WorkGroupIDX = true;140 141 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-y") ||142 !F.hasFnAttribute("amdgpu-no-cluster-id-y"))143 WorkGroupIDY = true;144 145 if (!F.hasFnAttribute("amdgpu-no-workgroup-id-z") ||146 !F.hasFnAttribute("amdgpu-no-cluster-id-z"))147 WorkGroupIDZ = true;148 }149 150 if (!AMDGPU::isGraphics(CC)) {151 if (IsKernel || !F.hasFnAttribute("amdgpu-no-workitem-id-x"))152 WorkItemIDX = true;153 154 if (!F.hasFnAttribute("amdgpu-no-workitem-id-y") &&155 ST.getMaxWorkitemID(F, 1) != 0)156 WorkItemIDY = true;157 158 if (!F.hasFnAttribute("amdgpu-no-workitem-id-z") &&159 ST.getMaxWorkitemID(F, 2) != 0)160 WorkItemIDZ = true;161 162 if (!IsKernel && !F.hasFnAttribute("amdgpu-no-lds-kernel-id"))163 LDSKernelId = true;164 }165 166 if (isEntryFunction()) {167 // X, XY, and XYZ are the only supported combinations, so make sure Y is168 // enabled if Z is.169 if (WorkItemIDZ)170 WorkItemIDY = true;171 172 if (!ST.flatScratchIsArchitected()) {173 PrivateSegmentWaveByteOffset = true;174 175 // HS and GS always have the scratch wave offset in SGPR5 on GFX9.176 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&177 (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))178 ArgInfo.PrivateSegmentWaveByteOffset =179 ArgDescriptor::createRegister(AMDGPU::SGPR5);180 }181 }182 183 Attribute A = F.getFnAttribute("amdgpu-git-ptr-high");184 StringRef S = A.getValueAsString();185 if (!S.empty())186 S.consumeInteger(0, GITPtrHigh);187 188 A = F.getFnAttribute("amdgpu-32bit-address-high-bits");189 S = A.getValueAsString();190 if (!S.empty())191 S.consumeInteger(0, HighBitsOf32BitAddress);192 193 MaxMemoryClusterDWords = F.getFnAttributeAsParsedInteger(194 "amdgpu-max-memory-cluster-dwords", DefaultMemoryClusterDWordsLimit);195 196 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch197 // VGPR available at all times. For now, reserve highest available VGPR. After198 // RA, shift it to the lowest available unused VGPR if the one exist.199 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {200 VGPRForAGPRCopy =201 AMDGPU::VGPR_32RegClass.getRegister(ST.getMaxNumVGPRs(F) - 1);202 }203 204 ClusterDims = AMDGPU::ClusterDimsAttr::get(F);205}206 207MachineFunctionInfo *SIMachineFunctionInfo::clone(208 BumpPtrAllocator &Allocator, MachineFunction &DestMF,209 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)210 const {211 return DestMF.cloneInfo<SIMachineFunctionInfo>(*this);212}213 214void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {215 limitOccupancy(getMaxWavesPerEU());216 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();217 limitOccupancy(ST.getOccupancyWithWorkGroupSizes(MF).second);218}219 220Register SIMachineFunctionInfo::addPrivateSegmentBuffer(221 const SIRegisterInfo &TRI) {222 ArgInfo.PrivateSegmentBuffer =223 ArgDescriptor::createRegister(TRI.getMatchingSuperReg(224 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SGPR_128RegClass));225 NumUserSGPRs += 4;226 return ArgInfo.PrivateSegmentBuffer.getRegister();227}228 229Register SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {230 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(231 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));232 NumUserSGPRs += 2;233 return ArgInfo.DispatchPtr.getRegister();234}235 236Register SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {237 ArgInfo.QueuePtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(238 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));239 NumUserSGPRs += 2;240 return ArgInfo.QueuePtr.getRegister();241}242 243Register SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {244 ArgInfo.KernargSegmentPtr245 = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(246 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));247 NumUserSGPRs += 2;248 return ArgInfo.KernargSegmentPtr.getRegister();249}250 251Register SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {252 ArgInfo.DispatchID = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(253 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));254 NumUserSGPRs += 2;255 return ArgInfo.DispatchID.getRegister();256}257 258Register SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {259 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(260 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));261 NumUserSGPRs += 2;262 return ArgInfo.FlatScratchInit.getRegister();263}264 265Register SIMachineFunctionInfo::addPrivateSegmentSize(const SIRegisterInfo &TRI) {266 ArgInfo.PrivateSegmentSize = ArgDescriptor::createRegister(getNextUserSGPR());267 NumUserSGPRs += 1;268 return ArgInfo.PrivateSegmentSize.getRegister();269}270 271Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {272 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(TRI.getMatchingSuperReg(273 getNextUserSGPR(), AMDGPU::sub0, &AMDGPU::SReg_64RegClass));274 NumUserSGPRs += 2;275 return ArgInfo.ImplicitBufferPtr.getRegister();276}277 278Register SIMachineFunctionInfo::addLDSKernelId() {279 ArgInfo.LDSKernelId = ArgDescriptor::createRegister(getNextUserSGPR());280 NumUserSGPRs += 1;281 return ArgInfo.LDSKernelId.getRegister();282}283 284SmallVectorImpl<MCRegister> *SIMachineFunctionInfo::addPreloadedKernArg(285 const SIRegisterInfo &TRI, const TargetRegisterClass *RC,286 unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs) {287 auto [It, Inserted] = ArgInfo.PreloadKernArgs.try_emplace(KernArgIdx);288 assert(Inserted && "Preload kernel argument allocated twice.");289 NumUserSGPRs += PaddingSGPRs;290 // If the available register tuples are aligned with the kernarg to be291 // preloaded use that register, otherwise we need to use a set of SGPRs and292 // merge them.293 if (!ArgInfo.FirstKernArgPreloadReg)294 ArgInfo.FirstKernArgPreloadReg = getNextUserSGPR();295 Register PreloadReg =296 TRI.getMatchingSuperReg(getNextUserSGPR(), AMDGPU::sub0, RC);297 auto &Regs = It->second.Regs;298 if (PreloadReg &&299 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {300 Regs.push_back(PreloadReg);301 NumUserSGPRs += AllocSizeDWord;302 } else {303 Regs.reserve(AllocSizeDWord);304 for (unsigned I = 0; I < AllocSizeDWord; ++I) {305 Regs.push_back(getNextUserSGPR());306 NumUserSGPRs++;307 }308 }309 310 // Track the actual number of SGPRs that HW will preload to.311 UserSGPRInfo.allocKernargPreloadSGPRs(AllocSizeDWord + PaddingSGPRs);312 return &Regs;313}314 315void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR,316 uint64_t Size, Align Alignment) {317 // Skip if it is an entry function or the register is already added.318 if (isEntryFunction() || WWMSpills.count(VGPR))319 return;320 321 // Skip if this is a function with the amdgpu_cs_chain or322 // amdgpu_cs_chain_preserve calling convention and this is a scratch register.323 // We never need to allocate a spill for these because we don't even need to324 // restore the inactive lanes for them (they're scratchier than the usual325 // scratch registers). We only need to do this if we have calls to326 // llvm.amdgcn.cs.chain (otherwise there's no one to save them for, since327 // chain functions do not return) and the function did not contain a call to328 // llvm.amdgcn.init.whole.wave (since in that case there are no inactive lanes329 // when entering the function).330 if (isChainFunction() &&331 (SIRegisterInfo::isChainScratchRegister(VGPR) ||332 !MF.getFrameInfo().hasTailCall() || hasInitWholeWave()))333 return;334 335 WWMSpills.insert(std::make_pair(336 VGPR, MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));337}338 339// Separate out the callee-saved and scratch registers.340void SIMachineFunctionInfo::splitWWMSpillRegisters(341 MachineFunction &MF,342 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,343 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const {344 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();345 for (auto &Reg : WWMSpills) {346 if (isCalleeSavedReg(CSRegs, Reg.first))347 CalleeSavedRegs.push_back(Reg);348 else349 ScratchRegs.push_back(Reg);350 }351}352 353bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs,354 MCPhysReg Reg) const {355 for (unsigned I = 0; CSRegs[I]; ++I) {356 if (CSRegs[I] == Reg)357 return true;358 }359 360 return false;361}362 363void SIMachineFunctionInfo::shiftWwmVGPRsToLowestRange(364 MachineFunction &MF, SmallVectorImpl<Register> &WWMVGPRs,365 BitVector &SavedVGPRs) {366 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();367 MachineRegisterInfo &MRI = MF.getRegInfo();368 for (unsigned I = 0, E = WWMVGPRs.size(); I < E; ++I) {369 Register Reg = WWMVGPRs[I];370 Register NewReg =371 TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);372 if (!NewReg || NewReg >= Reg)373 break;374 375 MRI.replaceRegWith(Reg, NewReg);376 377 // Update various tables with the new VGPR.378 WWMVGPRs[I] = NewReg;379 WWMReservedRegs.remove(Reg);380 WWMReservedRegs.insert(NewReg);381 MRI.reserveReg(NewReg, TRI);382 383 // Replace the register in SpillPhysVGPRs. This is needed to look for free384 // lanes while spilling special SGPRs like FP, BP, etc. during PEI.385 auto *RegItr = llvm::find(SpillPhysVGPRs, Reg);386 if (RegItr != SpillPhysVGPRs.end()) {387 unsigned Idx = std::distance(SpillPhysVGPRs.begin(), RegItr);388 SpillPhysVGPRs[Idx] = NewReg;389 }390 391 // The generic `determineCalleeSaves` might have set the old register if it392 // is in the CSR range.393 SavedVGPRs.reset(Reg);394 395 for (MachineBasicBlock &MBB : MF) {396 MBB.removeLiveIn(Reg);397 MBB.sortUniqueLiveIns();398 }399 400 Reg = NewReg;401 }402}403 404bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(405 MachineFunction &MF, int FI, unsigned LaneIndex) {406 MachineRegisterInfo &MRI = MF.getRegInfo();407 Register LaneVGPR;408 if (!LaneIndex) {409 LaneVGPR = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);410 SpillVGPRs.push_back(LaneVGPR);411 } else {412 LaneVGPR = SpillVGPRs.back();413 }414 415 SGPRSpillsToVirtualVGPRLanes[FI].emplace_back(LaneVGPR, LaneIndex);416 return true;417}418 419bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(420 MachineFunction &MF, int FI, unsigned LaneIndex, bool IsPrologEpilog) {421 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();422 const SIRegisterInfo *TRI = ST.getRegisterInfo();423 MachineRegisterInfo &MRI = MF.getRegInfo();424 Register LaneVGPR;425 if (!LaneIndex) {426 // Find the highest available register if called before RA to ensure the427 // lowest registers are available for allocation. The LaneVGPR, in that428 // case, will be shifted back to the lowest range after VGPR allocation.429 LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF,430 !IsPrologEpilog);431 if (LaneVGPR == AMDGPU::NoRegister) {432 // We have no VGPRs left for spilling SGPRs. Reset because we will not433 // partially spill the SGPR to VGPRs.434 SGPRSpillsToPhysicalVGPRLanes.erase(FI);435 return false;436 }437 438 if (IsPrologEpilog)439 allocateWWMSpill(MF, LaneVGPR);440 441 reserveWWMRegister(LaneVGPR);442 for (MachineBasicBlock &MBB : MF) {443 MBB.addLiveIn(LaneVGPR);444 MBB.sortUniqueLiveIns();445 }446 SpillPhysVGPRs.push_back(LaneVGPR);447 } else {448 LaneVGPR = SpillPhysVGPRs.back();449 }450 451 SGPRSpillsToPhysicalVGPRLanes[FI].emplace_back(LaneVGPR, LaneIndex);452 return true;453}454 455bool SIMachineFunctionInfo::allocateSGPRSpillToVGPRLane(456 MachineFunction &MF, int FI, bool SpillToPhysVGPRLane,457 bool IsPrologEpilog) {458 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =459 SpillToPhysVGPRLane ? SGPRSpillsToPhysicalVGPRLanes[FI]460 : SGPRSpillsToVirtualVGPRLanes[FI];461 462 // This has already been allocated.463 if (!SpillLanes.empty())464 return true;465 466 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();467 MachineFrameInfo &FrameInfo = MF.getFrameInfo();468 unsigned WaveSize = ST.getWavefrontSize();469 470 unsigned Size = FrameInfo.getObjectSize(FI);471 unsigned NumLanes = Size / 4;472 473 if (NumLanes > WaveSize)474 return false;475 476 assert(Size >= 4 && "invalid sgpr spill size");477 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&478 "not spilling SGPRs to VGPRs");479 480 unsigned &NumSpillLanes = SpillToPhysVGPRLane ? NumPhysicalVGPRSpillLanes481 : NumVirtualVGPRSpillLanes;482 483 for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) {484 unsigned LaneIndex = (NumSpillLanes % WaveSize);485 486 bool Allocated = SpillToPhysVGPRLane487 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex,488 IsPrologEpilog)489 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);490 if (!Allocated) {491 NumSpillLanes -= I;492 return false;493 }494 }495 496 return true;497}498 499/// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.500/// Either AGPR is spilled to VGPR to vice versa.501/// Returns true if a \p FI can be eliminated completely.502bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF,503 int FI,504 bool isAGPRtoVGPR) {505 MachineRegisterInfo &MRI = MF.getRegInfo();506 MachineFrameInfo &FrameInfo = MF.getFrameInfo();507 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();508 509 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));510 511 auto &Spill = VGPRToAGPRSpills[FI];512 513 // This has already been allocated.514 if (!Spill.Lanes.empty())515 return Spill.FullyAllocated;516 517 unsigned Size = FrameInfo.getObjectSize(FI);518 unsigned NumLanes = Size / 4;519 Spill.Lanes.resize(NumLanes, AMDGPU::NoRegister);520 521 const TargetRegisterClass &RC =522 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;523 auto Regs = RC.getRegisters();524 525 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;526 const SIRegisterInfo *TRI = ST.getRegisterInfo();527 Spill.FullyAllocated = true;528 529 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize530 // once.531 BitVector OtherUsedRegs;532 OtherUsedRegs.resize(TRI->getNumRegs());533 534 const uint32_t *CSRMask =535 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());536 if (CSRMask)537 OtherUsedRegs.setBitsInMask(CSRMask);538 539 // TODO: Should include register tuples, but doesn't matter with current540 // usage.541 for (MCPhysReg Reg : SpillAGPR)542 OtherUsedRegs.set(Reg);543 for (MCPhysReg Reg : SpillVGPR)544 OtherUsedRegs.set(Reg);545 546 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();547 for (int I = NumLanes - 1; I >= 0; --I) {548 NextSpillReg = std::find_if(549 NextSpillReg, Regs.end(), [&MRI, &OtherUsedRegs](MCPhysReg Reg) {550 return MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg) &&551 !OtherUsedRegs[Reg];552 });553 554 if (NextSpillReg == Regs.end()) { // Registers exhausted555 Spill.FullyAllocated = false;556 break;557 }558 559 OtherUsedRegs.set(*NextSpillReg);560 SpillRegs.push_back(*NextSpillReg);561 MRI.reserveReg(*NextSpillReg, TRI);562 Spill.Lanes[I] = *NextSpillReg++;563 }564 565 return Spill.FullyAllocated;566}567 568bool SIMachineFunctionInfo::removeDeadFrameIndices(569 MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) {570 // Remove dead frame indices from function frame, however keep FP & BP since571 // spills for them haven't been inserted yet. And also make sure to remove the572 // frame indices from `SGPRSpillsToVirtualVGPRLanes` data structure,573 // otherwise, it could result in an unexpected side effect and bug, in case of574 // any re-mapping of freed frame indices by later pass(es) like "stack slot575 // coloring".576 for (auto &R : make_early_inc_range(SGPRSpillsToVirtualVGPRLanes)) {577 MFI.RemoveStackObject(R.first);578 SGPRSpillsToVirtualVGPRLanes.erase(R.first);579 }580 581 // Remove the dead frame indices of CSR SGPRs which are spilled to physical582 // VGPR lanes during SILowerSGPRSpills pass.583 if (!ResetSGPRSpillStackIDs) {584 for (auto &R : make_early_inc_range(SGPRSpillsToPhysicalVGPRLanes)) {585 MFI.RemoveStackObject(R.first);586 SGPRSpillsToPhysicalVGPRLanes.erase(R.first);587 }588 }589 bool HaveSGPRToMemory = false;590 591 if (ResetSGPRSpillStackIDs) {592 // All other SGPRs must be allocated on the default stack, so reset the593 // stack ID.594 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E;595 ++I) {596 if (!checkIndexInPrologEpilogSGPRSpills(I)) {597 if (MFI.getStackID(I) == TargetStackID::SGPRSpill) {598 MFI.setStackID(I, TargetStackID::Default);599 HaveSGPRToMemory = true;600 }601 }602 }603 }604 605 for (auto &R : VGPRToAGPRSpills) {606 if (R.second.IsDead)607 MFI.RemoveStackObject(R.first);608 }609 610 return HaveSGPRToMemory;611}612 613int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI,614 const SIRegisterInfo &TRI) {615 if (ScavengeFI)616 return *ScavengeFI;617 618 ScavengeFI =619 MFI.CreateStackObject(TRI.getSpillSize(AMDGPU::SGPR_32RegClass),620 TRI.getSpillAlign(AMDGPU::SGPR_32RegClass), false);621 return *ScavengeFI;622}623 624MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {625 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");626 return AMDGPU::SGPR0 + NumUserSGPRs;627}628 629MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {630 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;631}632 633void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) {634 VRegFlags.grow(Reg);635}636 637void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg,638 Register SrcReg) {639 VRegFlags.grow(NewReg);640 VRegFlags[NewReg] = VRegFlags[SrcReg];641}642 643Register644SIMachineFunctionInfo::getGITPtrLoReg(const MachineFunction &MF) const {645 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();646 if (!ST.isAmdPalOS())647 return Register();648 Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in649 if (ST.hasMergedShaders()) {650 switch (MF.getFunction().getCallingConv()) {651 case CallingConv::AMDGPU_HS:652 case CallingConv::AMDGPU_GS:653 // Low GIT address is passed in s8 rather than s0 for an LS+HS or654 // ES+GS merged shader on gfx9+.655 GitPtrLo = AMDGPU::SGPR8;656 return GitPtrLo;657 default:658 return GitPtrLo;659 }660 }661 return GitPtrLo;662}663 664static yaml::StringValue regToString(Register Reg,665 const TargetRegisterInfo &TRI) {666 yaml::StringValue Dest;667 {668 raw_string_ostream OS(Dest.Value);669 OS << printReg(Reg, &TRI);670 }671 return Dest;672}673 674static std::optional<yaml::SIArgumentInfo>675convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,676 const TargetRegisterInfo &TRI) {677 yaml::SIArgumentInfo AI;678 679 auto convertArg = [&](std::optional<yaml::SIArgument> &A,680 const ArgDescriptor &Arg) {681 if (!Arg)682 return false;683 684 // Create a register or stack argument.685 yaml::SIArgument SA = yaml::SIArgument::createArgument(Arg.isRegister());686 if (Arg.isRegister()) {687 raw_string_ostream OS(SA.RegisterName.Value);688 OS << printReg(Arg.getRegister(), &TRI);689 } else690 SA.StackOffset = Arg.getStackOffset();691 // Check and update the optional mask.692 if (Arg.isMasked())693 SA.Mask = Arg.getMask();694 695 A = SA;696 return true;697 };698 699 bool Any = false;700 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);701 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);702 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);703 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);704 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);705 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);706 Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);707 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);708 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);709 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);710 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);711 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);712 Any |= convertArg(AI.PrivateSegmentWaveByteOffset,713 ArgInfo.PrivateSegmentWaveByteOffset);714 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);715 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);716 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);717 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);718 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);719 720 // Write FirstKernArgPreloadReg separately, since it's a Register,721 // not ArgDescriptor.722 if (ArgInfo.FirstKernArgPreloadReg) {723 Register Reg = ArgInfo.FirstKernArgPreloadReg;724 assert(Reg.isPhysical() &&725 "FirstKernArgPreloadReg must be a physical register");726 727 yaml::SIArgument SA = yaml::SIArgument::createArgument(true);728 raw_string_ostream OS(SA.RegisterName.Value);729 OS << printReg(Reg, &TRI);730 731 AI.FirstKernArgPreloadReg = SA;732 Any = true;733 }734 735 if (Any)736 return AI;737 738 return std::nullopt;739}740 741yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(742 const llvm::SIMachineFunctionInfo &MFI, const TargetRegisterInfo &TRI,743 const llvm::MachineFunction &MF)744 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),745 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),746 GDSSize(MFI.getGDSSize()), DynLDSAlign(MFI.getDynLDSAlign()),747 IsEntryFunction(MFI.isEntryFunction()),748 NoSignedZerosFPMath(MFI.hasNoSignedZerosFPMath()),749 MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),750 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),751 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),752 NumWaveDispatchSGPRs(MFI.getNumWaveDispatchSGPRs()),753 NumWaveDispatchVGPRs(MFI.getNumWaveDispatchVGPRs()),754 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),755 Occupancy(MFI.getOccupancy()),756 ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),757 FrameOffsetReg(regToString(MFI.getFrameOffsetReg(), TRI)),758 StackPtrOffsetReg(regToString(MFI.getStackPtrOffsetReg(), TRI)),759 BytesInStackArgArea(MFI.getBytesInStackArgArea()),760 ReturnsVoid(MFI.returnsVoid()),761 ArgInfo(convertArgumentInfo(MFI.getArgInfo(), TRI)),762 PSInputAddr(MFI.getPSInputAddr()), PSInputEnable(MFI.getPSInputEnable()),763 MaxMemoryClusterDWords(MFI.getMaxMemoryClusterDWords()),764 Mode(MFI.getMode()), HasInitWholeWave(MFI.hasInitWholeWave()),765 IsWholeWaveFunction(MFI.isWholeWaveFunction()),766 DynamicVGPRBlockSize(MFI.getDynamicVGPRBlockSize()),767 ScratchReservedForDynamicVGPRs(MFI.getScratchReservedForDynamicVGPRs()),768 NumKernargPreloadSGPRs(MFI.getNumKernargPreloadedSGPRs()) {769 for (Register Reg : MFI.getSGPRSpillPhysVGPRs())770 SpillPhysVGPRS.push_back(regToString(Reg, TRI));771 772 for (Register Reg : MFI.getWWMReservedRegs())773 WWMReservedRegs.push_back(regToString(Reg, TRI));774 775 if (MFI.getLongBranchReservedReg())776 LongBranchReservedReg = regToString(MFI.getLongBranchReservedReg(), TRI);777 if (MFI.getVGPRForAGPRCopy())778 VGPRForAGPRCopy = regToString(MFI.getVGPRForAGPRCopy(), TRI);779 780 if (MFI.getSGPRForEXECCopy())781 SGPRForEXECCopy = regToString(MFI.getSGPRForEXECCopy(), TRI);782 783 auto SFI = MFI.getOptionalScavengeFI();784 if (SFI)785 ScavengeFI = yaml::FrameIndex(*SFI, MF.getFrameInfo());786}787 788void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {789 MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, *this);790}791 792bool SIMachineFunctionInfo::initializeBaseYamlFields(793 const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF,794 PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) {795 ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;796 MaxKernArgAlign = YamlMFI.MaxKernArgAlign;797 LDSSize = YamlMFI.LDSSize;798 GDSSize = YamlMFI.GDSSize;799 DynLDSAlign = YamlMFI.DynLDSAlign;800 PSInputAddr = YamlMFI.PSInputAddr;801 PSInputEnable = YamlMFI.PSInputEnable;802 MaxMemoryClusterDWords = YamlMFI.MaxMemoryClusterDWords;803 HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress;804 Occupancy = YamlMFI.Occupancy;805 IsEntryFunction = YamlMFI.IsEntryFunction;806 NoSignedZerosFPMath = YamlMFI.NoSignedZerosFPMath;807 MemoryBound = YamlMFI.MemoryBound;808 WaveLimiter = YamlMFI.WaveLimiter;809 HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;810 HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;811 NumWaveDispatchSGPRs = YamlMFI.NumWaveDispatchSGPRs;812 NumWaveDispatchVGPRs = YamlMFI.NumWaveDispatchVGPRs;813 BytesInStackArgArea = YamlMFI.BytesInStackArgArea;814 ReturnsVoid = YamlMFI.ReturnsVoid;815 IsWholeWaveFunction = YamlMFI.IsWholeWaveFunction;816 817 UserSGPRInfo.allocKernargPreloadSGPRs(YamlMFI.NumKernargPreloadSGPRs);818 819 if (YamlMFI.ScavengeFI) {820 auto FIOrErr = YamlMFI.ScavengeFI->getFI(MF.getFrameInfo());821 if (!FIOrErr) {822 // Create a diagnostic for a the frame index.823 const MemoryBuffer &Buffer =824 *PFS.SM->getMemoryBuffer(PFS.SM->getMainFileID());825 826 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1,827 SourceMgr::DK_Error, toString(FIOrErr.takeError()),828 "", {}, {});829 SourceRange = YamlMFI.ScavengeFI->SourceRange;830 return true;831 }832 ScavengeFI = *FIOrErr;833 } else {834 ScavengeFI = std::nullopt;835 }836 return false;837}838 839bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const {840 auto [MinNumAGPR, MaxNumAGPR] =841 AMDGPU::getIntegerPairAttribute(F, "amdgpu-agpr-alloc", {~0u, ~0u},842 /*OnlyFirstRequired=*/true);843 return MinNumAGPR != 0u;844}845