587 lines · cpp
1//===-- SILowerSGPRSPills.cpp ---------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Handle SGPR spills. This pass takes the place of PrologEpilogInserter for all10// SGPR spills, so must insert CSR SGPR spills as well as expand them.11//12// This pass must never create new SGPR virtual registers.13//14// FIXME: Must stop RegScavenger spills in later passes.15//16//===----------------------------------------------------------------------===//17 18#include "SILowerSGPRSpills.h"19#include "AMDGPU.h"20#include "GCNSubtarget.h"21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"22#include "SIMachineFunctionInfo.h"23#include "llvm/CodeGen/LiveIntervals.h"24#include "llvm/CodeGen/MachineDominators.h"25#include "llvm/CodeGen/MachineFrameInfo.h"26#include "llvm/CodeGen/RegisterScavenging.h"27#include "llvm/InitializePasses.h"28 29using namespace llvm;30 31#define DEBUG_TYPE "si-lower-sgpr-spills"32 33using MBBVector = SmallVector<MachineBasicBlock *, 4>;34 35namespace {36 37static cl::opt<unsigned> MaxNumVGPRsForWwmAllocation(38 "amdgpu-num-vgprs-for-wwm-alloc",39 cl::desc("Max num VGPRs for whole-wave register allocation."),40 cl::ReallyHidden, cl::init(10));41 42class SILowerSGPRSpills {43private:44 const SIRegisterInfo *TRI = nullptr;45 const SIInstrInfo *TII = nullptr;46 LiveIntervals *LIS = nullptr;47 SlotIndexes *Indexes = nullptr;48 MachineDominatorTree *MDT = nullptr;49 50 // Save and Restore blocks of the current function. Typically there is a51 // single save block, unless Windows EH funclets are involved.52 MBBVector SaveBlocks;53 MBBVector RestoreBlocks;54 55public:56 SILowerSGPRSpills(LiveIntervals *LIS, SlotIndexes *Indexes,57 MachineDominatorTree *MDT)58 : LIS(LIS), Indexes(Indexes), MDT(MDT) {}59 bool run(MachineFunction &MF);60 void calculateSaveRestoreBlocks(MachineFunction &MF);61 bool spillCalleeSavedRegs(MachineFunction &MF,62 SmallVectorImpl<int> &CalleeSavedFIs);63 void updateLaneVGPRDomInstr(64 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,65 DenseMap<Register, MachineBasicBlock::iterator> &LaneVGPRDomInstr);66 void determineRegsForWWMAllocation(MachineFunction &MF, BitVector &RegMask);67};68 69class SILowerSGPRSpillsLegacy : public MachineFunctionPass {70public:71 static char ID;72 73 SILowerSGPRSpillsLegacy() : MachineFunctionPass(ID) {}74 75 bool runOnMachineFunction(MachineFunction &MF) override;76 77 void getAnalysisUsage(AnalysisUsage &AU) const override {78 AU.addRequired<MachineDominatorTreeWrapperPass>();79 AU.setPreservesAll();80 MachineFunctionPass::getAnalysisUsage(AU);81 }82 83 MachineFunctionProperties getClearedProperties() const override {84 // SILowerSGPRSpills introduces new Virtual VGPRs for spilling SGPRs.85 return MachineFunctionProperties().setIsSSA().setNoVRegs();86 }87};88 89} // end anonymous namespace90 91char SILowerSGPRSpillsLegacy::ID = 0;92 93INITIALIZE_PASS_BEGIN(SILowerSGPRSpillsLegacy, DEBUG_TYPE,94 "SI lower SGPR spill instructions", false, false)95INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)96INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)97INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)98INITIALIZE_PASS_END(SILowerSGPRSpillsLegacy, DEBUG_TYPE,99 "SI lower SGPR spill instructions", false, false)100 101char &llvm::SILowerSGPRSpillsLegacyID = SILowerSGPRSpillsLegacy::ID;102 103static bool isLiveIntoMBB(MCRegister Reg, MachineBasicBlock &MBB,104 const TargetRegisterInfo *TRI) {105 for (MCRegAliasIterator R(Reg, TRI, true); R.isValid(); ++R) {106 if (MBB.isLiveIn(*R)) {107 return true;108 }109 }110 return false;111}112 113/// Insert spill code for the callee-saved registers used in the function.114static void insertCSRSaves(MachineBasicBlock &SaveBlock,115 ArrayRef<CalleeSavedInfo> CSI, SlotIndexes *Indexes,116 LiveIntervals *LIS) {117 MachineFunction &MF = *SaveBlock.getParent();118 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();119 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();120 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();121 const SIRegisterInfo *RI = ST.getRegisterInfo();122 123 MachineBasicBlock::iterator I = SaveBlock.begin();124 if (!TFI->spillCalleeSavedRegisters(SaveBlock, I, CSI, RI)) {125 for (const CalleeSavedInfo &CS : CSI) {126 // Insert the spill to the stack frame.127 MCRegister Reg = CS.getReg();128 129 MachineInstrSpan MIS(I, &SaveBlock);130 const TargetRegisterClass *RC = RI->getMinimalPhysRegClass(131 Reg, Reg == RI->getReturnAddressReg(MF) ? MVT::i64 : MVT::i32);132 133 // If this value was already livein, we probably have a direct use of the134 // incoming register value, so don't kill at the spill point. This happens135 // since we pass some special inputs (workgroup IDs) in the callee saved136 // range.137 const bool IsLiveIn = isLiveIntoMBB(Reg, SaveBlock, RI);138 TII.storeRegToStackSlot(SaveBlock, I, Reg, !IsLiveIn, CS.getFrameIdx(),139 RC, Register());140 141 if (Indexes) {142 assert(std::distance(MIS.begin(), I) == 1);143 MachineInstr &Inst = *std::prev(I);144 Indexes->insertMachineInstrInMaps(Inst);145 }146 147 if (LIS)148 LIS->removeAllRegUnitsForPhysReg(Reg);149 }150 } else {151 // TFI doesn't update Indexes and LIS, so we have to do it separately.152 if (Indexes)153 Indexes->repairIndexesInRange(&SaveBlock, SaveBlock.begin(), I);154 155 if (LIS)156 for (const CalleeSavedInfo &CS : CSI)157 LIS->removeAllRegUnitsForPhysReg(CS.getReg());158 }159}160 161/// Insert restore code for the callee-saved registers used in the function.162static void insertCSRRestores(MachineBasicBlock &RestoreBlock,163 MutableArrayRef<CalleeSavedInfo> CSI,164 SlotIndexes *Indexes, LiveIntervals *LIS) {165 MachineFunction &MF = *RestoreBlock.getParent();166 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();167 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();168 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();169 // Restore all registers immediately before the return and any170 // terminators that precede it.171 MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator();172 const MachineBasicBlock::iterator BeforeRestoresI =173 I == RestoreBlock.begin() ? I : std::prev(I);174 175 // FIXME: Just emit the readlane/writelane directly176 if (!TFI->restoreCalleeSavedRegisters(RestoreBlock, I, CSI, TRI)) {177 for (const CalleeSavedInfo &CI : reverse(CSI)) {178 // Insert in reverse order. loadRegFromStackSlot can insert179 // multiple instructions.180 TFI->restoreCalleeSavedRegister(RestoreBlock, I, CI, &TII, TRI);181 182 if (Indexes) {183 MachineInstr &Inst = *std::prev(I);184 Indexes->insertMachineInstrInMaps(Inst);185 }186 187 if (LIS)188 LIS->removeAllRegUnitsForPhysReg(CI.getReg());189 }190 } else {191 // TFI doesn't update Indexes and LIS, so we have to do it separately.192 if (Indexes)193 Indexes->repairIndexesInRange(&RestoreBlock, BeforeRestoresI,194 RestoreBlock.getFirstTerminator());195 196 if (LIS)197 for (const CalleeSavedInfo &CS : CSI)198 LIS->removeAllRegUnitsForPhysReg(CS.getReg());199 }200}201 202/// Compute the sets of entry and return blocks for saving and restoring203/// callee-saved registers, and placing prolog and epilog code.204void SILowerSGPRSpills::calculateSaveRestoreBlocks(MachineFunction &MF) {205 const MachineFrameInfo &MFI = MF.getFrameInfo();206 207 // Even when we do not change any CSR, we still want to insert the208 // prologue and epilogue of the function.209 // So set the save points for those.210 211 // Use the points found by shrink-wrapping, if any.212 if (!MFI.getSavePoints().empty()) {213 assert(MFI.getSavePoints().size() == 1 &&214 "Multiple save points not yet supported!");215 const auto &SavePoint = *MFI.getSavePoints().begin();216 SaveBlocks.push_back(SavePoint.first);217 assert(MFI.getRestorePoints().size() == 1 &&218 "Multiple restore points not yet supported!");219 const auto &RestorePoint = *MFI.getRestorePoints().begin();220 MachineBasicBlock *RestoreBlock = RestorePoint.first;221 // If RestoreBlock does not have any successor and is not a return block222 // then the end point is unreachable and we do not need to insert any223 // epilogue.224 if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())225 RestoreBlocks.push_back(RestoreBlock);226 return;227 }228 229 // Save refs to entry and return blocks.230 SaveBlocks.push_back(&MF.front());231 for (MachineBasicBlock &MBB : MF) {232 if (MBB.isEHFuncletEntry())233 SaveBlocks.push_back(&MBB);234 if (MBB.isReturnBlock())235 RestoreBlocks.push_back(&MBB);236 }237}238 239// TODO: To support shrink wrapping, this would need to copy240// PrologEpilogInserter's updateLiveness.241static void updateLiveness(MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI) {242 MachineBasicBlock &EntryBB = MF.front();243 244 for (const CalleeSavedInfo &CSIReg : CSI)245 EntryBB.addLiveIn(CSIReg.getReg());246 EntryBB.sortUniqueLiveIns();247}248 249bool SILowerSGPRSpills::spillCalleeSavedRegs(250 MachineFunction &MF, SmallVectorImpl<int> &CalleeSavedFIs) {251 MachineRegisterInfo &MRI = MF.getRegInfo();252 const Function &F = MF.getFunction();253 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();254 const SIFrameLowering *TFI = ST.getFrameLowering();255 MachineFrameInfo &MFI = MF.getFrameInfo();256 RegScavenger *RS = nullptr;257 258 // Determine which of the registers in the callee save list should be saved.259 BitVector SavedRegs;260 TFI->determineCalleeSavesSGPR(MF, SavedRegs, RS);261 262 // Add the code to save and restore the callee saved registers.263 if (!F.hasFnAttribute(Attribute::Naked)) {264 // FIXME: This is a lie. The CalleeSavedInfo is incomplete, but this is265 // necessary for verifier liveness checks.266 MFI.setCalleeSavedInfoValid(true);267 268 std::vector<CalleeSavedInfo> CSI;269 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();270 271 for (unsigned I = 0; CSRegs[I]; ++I) {272 MCRegister Reg = CSRegs[I];273 274 if (SavedRegs.test(Reg)) {275 const TargetRegisterClass *RC =276 TRI->getMinimalPhysRegClass(Reg, MVT::i32);277 int JunkFI = MFI.CreateStackObject(TRI->getSpillSize(*RC),278 TRI->getSpillAlign(*RC), true);279 280 CSI.emplace_back(Reg, JunkFI);281 CalleeSavedFIs.push_back(JunkFI);282 }283 }284 285 if (!CSI.empty()) {286 for (MachineBasicBlock *SaveBlock : SaveBlocks)287 insertCSRSaves(*SaveBlock, CSI, Indexes, LIS);288 289 // Add live ins to save blocks.290 assert(SaveBlocks.size() == 1 && "shrink wrapping not fully implemented");291 updateLiveness(MF, CSI);292 293 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)294 insertCSRRestores(*RestoreBlock, CSI, Indexes, LIS);295 return true;296 }297 }298 299 return false;300}301 302void SILowerSGPRSpills::updateLaneVGPRDomInstr(303 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,304 DenseMap<Register, MachineBasicBlock::iterator> &LaneVGPRDomInstr) {305 // For the Def of a virtual LaneVPGR to dominate all its uses, we should306 // insert an IMPLICIT_DEF before the dominating spill. Switching to a307 // depth first order doesn't really help since the machine function can be in308 // the unstructured control flow post-SSA. For each virtual register, hence309 // finding the common dominator to get either the dominating spill or a block310 // dominating all spills.311 SIMachineFunctionInfo *FuncInfo =312 MBB->getParent()->getInfo<SIMachineFunctionInfo>();313 ArrayRef<SIRegisterInfo::SpilledReg> VGPRSpills =314 FuncInfo->getSGPRSpillToVirtualVGPRLanes(FI);315 Register PrevLaneVGPR;316 for (auto &Spill : VGPRSpills) {317 if (PrevLaneVGPR == Spill.VGPR)318 continue;319 320 PrevLaneVGPR = Spill.VGPR;321 auto I = LaneVGPRDomInstr.find(Spill.VGPR);322 if (Spill.Lane == 0 && I == LaneVGPRDomInstr.end()) {323 // Initially add the spill instruction itself for Insertion point.324 LaneVGPRDomInstr[Spill.VGPR] = InsertPt;325 } else {326 assert(I != LaneVGPRDomInstr.end());327 auto PrevInsertPt = I->second;328 MachineBasicBlock *DomMBB = PrevInsertPt->getParent();329 if (DomMBB == MBB) {330 // The insertion point earlier selected in a predecessor block whose331 // spills are currently being lowered. The earlier InsertPt would be332 // the one just before the block terminator and it should be changed333 // if we insert any new spill in it.334 if (MDT->dominates(&*InsertPt, &*PrevInsertPt))335 I->second = InsertPt;336 337 continue;338 }339 340 // Find the common dominator block between PrevInsertPt and the341 // current spill.342 DomMBB = MDT->findNearestCommonDominator(DomMBB, MBB);343 if (DomMBB == MBB)344 I->second = InsertPt;345 else if (DomMBB != PrevInsertPt->getParent())346 I->second = &(*DomMBB->getFirstTerminator());347 }348 }349}350 351void SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF,352 BitVector &RegMask) {353 // Determine an optimal number of VGPRs for WWM allocation. The complement354 // list will be available for allocating other VGPR virtual registers.355 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();356 MachineRegisterInfo &MRI = MF.getRegInfo();357 BitVector ReservedRegs = TRI->getReservedRegs(MF);358 BitVector NonWwmAllocMask(TRI->getNumRegs());359 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();360 361 // FIXME: MaxNumVGPRsForWwmAllocation might need to be adjusted in the future362 // to have a balanced allocation between WWM values and per-thread vector363 // register operands.364 unsigned NumRegs = MaxNumVGPRsForWwmAllocation;365 NumRegs =366 std::min(static_cast<unsigned>(MFI->getSGPRSpillVGPRs().size()), NumRegs);367 368 auto [MaxNumVGPRs, MaxNumAGPRs] = ST.getMaxNumVectorRegs(MF.getFunction());369 // Try to use the highest available registers for now. Later after370 // vgpr-regalloc, they can be shifted to the lowest range.371 unsigned I = 0;372 for (unsigned Reg = AMDGPU::VGPR0 + MaxNumVGPRs - 1;373 (I < NumRegs) && (Reg >= AMDGPU::VGPR0); --Reg) {374 if (!ReservedRegs.test(Reg) &&375 !MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true)) {376 TRI->markSuperRegs(RegMask, Reg);377 ++I;378 }379 }380 381 if (I != NumRegs) {382 // Reserve an arbitrary register and report the error.383 TRI->markSuperRegs(RegMask, AMDGPU::VGPR0);384 MF.getFunction().getContext().emitError(385 "cannot find enough VGPRs for wwm-regalloc");386 }387}388 389bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) {390 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();391 LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;392 auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();393 SlotIndexes *Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;394 MachineDominatorTree *MDT =395 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();396 return SILowerSGPRSpills(LIS, Indexes, MDT).run(MF);397}398 399bool SILowerSGPRSpills::run(MachineFunction &MF) {400 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();401 TII = ST.getInstrInfo();402 TRI = &TII->getRegisterInfo();403 404 assert(SaveBlocks.empty() && RestoreBlocks.empty());405 406 // First, expose any CSR SGPR spills. This is mostly the same as what PEI407 // does, but somewhat simpler.408 calculateSaveRestoreBlocks(MF);409 SmallVector<int> CalleeSavedFIs;410 bool HasCSRs = spillCalleeSavedRegs(MF, CalleeSavedFIs);411 412 MachineFrameInfo &MFI = MF.getFrameInfo();413 MachineRegisterInfo &MRI = MF.getRegInfo();414 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();415 416 if (!MFI.hasStackObjects() && !HasCSRs) {417 SaveBlocks.clear();418 RestoreBlocks.clear();419 return false;420 }421 422 bool MadeChange = false;423 bool SpilledToVirtVGPRLanes = false;424 425 // TODO: CSR VGPRs will never be spilled to AGPRs. These can probably be426 // handled as SpilledToReg in regular PrologEpilogInserter.427 const bool HasSGPRSpillToVGPR = TRI->spillSGPRToVGPR() &&428 (HasCSRs || FuncInfo->hasSpilledSGPRs());429 if (HasSGPRSpillToVGPR) {430 // Process all SGPR spills before frame offsets are finalized. Ideally SGPRs431 // are spilled to VGPRs, in which case we can eliminate the stack usage.432 //433 // This operates under the assumption that only other SGPR spills are users434 // of the frame index.435 436 // To track the spill frame indices handled in this pass.437 BitVector SpillFIs(MFI.getObjectIndexEnd(), false);438 439 // To track the IMPLICIT_DEF insertion point for the lane vgprs.440 DenseMap<Register, MachineBasicBlock::iterator> LaneVGPRDomInstr;441 442 for (MachineBasicBlock &MBB : MF) {443 for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {444 if (!TII->isSGPRSpill(MI))445 continue;446 447 if (MI.getOperand(0).isUndef()) {448 if (Indexes)449 Indexes->removeMachineInstrFromMaps(MI);450 MI.eraseFromParent();451 continue;452 }453 454 int FI = TII->getNamedOperand(MI, AMDGPU::OpName::addr)->getIndex();455 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);456 457 bool IsCalleeSaveSGPRSpill = llvm::is_contained(CalleeSavedFIs, FI);458 if (IsCalleeSaveSGPRSpill) {459 // Spill callee-saved SGPRs into physical VGPR lanes.460 461 // TODO: This is to ensure the CFIs are static for efficient frame462 // unwinding in the debugger. Spilling them into virtual VGPR lanes463 // involve regalloc to allocate the physical VGPRs and that might464 // cause intermediate spill/split of such liveranges for successful465 // allocation. This would result in broken CFI encoding unless the466 // regalloc aware CFI generation to insert new CFIs along with the467 // intermediate spills is implemented. There is no such support468 // currently exist in the LLVM compiler.469 if (FuncInfo->allocateSGPRSpillToVGPRLane(470 MF, FI, /*SpillToPhysVGPRLane=*/true)) {471 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(472 MI, FI, nullptr, Indexes, LIS, true);473 if (!Spilled)474 llvm_unreachable(475 "failed to spill SGPR to physical VGPR lane when allocated");476 }477 } else {478 MachineInstrSpan MIS(&MI, &MBB);479 if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) {480 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(481 MI, FI, nullptr, Indexes, LIS);482 if (!Spilled)483 llvm_unreachable(484 "failed to spill SGPR to virtual VGPR lane when allocated");485 SpillFIs.set(FI);486 updateLaneVGPRDomInstr(FI, &MBB, MIS.begin(), LaneVGPRDomInstr);487 SpilledToVirtVGPRLanes = true;488 }489 }490 }491 }492 493 for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {494 auto InsertPt = LaneVGPRDomInstr[Reg];495 // Insert the IMPLICIT_DEF at the identified points.496 MachineBasicBlock &Block = *InsertPt->getParent();497 DebugLoc DL = Block.findDebugLoc(InsertPt);498 auto MIB =499 BuildMI(Block, *InsertPt, DL, TII->get(AMDGPU::IMPLICIT_DEF), Reg);500 501 // Add WWM flag to the virtual register.502 FuncInfo->setFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG);503 504 // Set SGPR_SPILL asm printer flag505 MIB->setAsmPrinterFlag(AMDGPU::SGPR_SPILL);506 if (LIS) {507 LIS->InsertMachineInstrInMaps(*MIB);508 LIS->createAndComputeVirtRegInterval(Reg);509 }510 }511 512 // Determine the registers for WWM allocation and also compute the register513 // mask for non-wwm VGPR allocation.514 if (FuncInfo->getSGPRSpillVGPRs().size()) {515 BitVector WwmRegMask(TRI->getNumRegs());516 517 determineRegsForWWMAllocation(MF, WwmRegMask);518 519 BitVector NonWwmRegMask(WwmRegMask);520 NonWwmRegMask.flip().clearBitsNotInMask(TRI->getAllVGPRRegMask());521 522 // The complement set will be the registers for non-wwm (per-thread) vgpr523 // allocation.524 FuncInfo->updateNonWWMRegMask(NonWwmRegMask);525 }526 527 for (MachineBasicBlock &MBB : MF) {528 // FIXME: The dead frame indices are replaced with a null register from529 // the debug value instructions. We should instead, update it with the530 // correct register value. But not sure the register value alone is531 // adequate to lower the DIExpression. It should be worked out later.532 for (MachineInstr &MI : MBB) {533 if (MI.isDebugValue()) {534 uint32_t StackOperandIdx = MI.isDebugValueList() ? 2 : 0;535 if (MI.getOperand(StackOperandIdx).isFI() &&536 !MFI.isFixedObjectIndex(537 MI.getOperand(StackOperandIdx).getIndex()) &&538 SpillFIs[MI.getOperand(StackOperandIdx).getIndex()]) {539 MI.getOperand(StackOperandIdx)540 .ChangeToRegister(Register(), false /*isDef*/);541 }542 }543 }544 }545 546 // All those frame indices which are dead by now should be removed from the547 // function frame. Otherwise, there is a side effect such as re-mapping of548 // free frame index ids by the later pass(es) like "stack slot coloring"549 // which in turn could mess-up with the book keeping of "frame index to VGPR550 // lane".551 FuncInfo->removeDeadFrameIndices(MFI, /*ResetSGPRSpillStackIDs*/ false);552 553 MadeChange = true;554 }555 556 if (SpilledToVirtVGPRLanes) {557 const TargetRegisterClass *RC = TRI->getWaveMaskRegClass();558 // Shift back the reserved SGPR for EXEC copy into the lowest range.559 // This SGPR is reserved to handle the whole-wave spill/copy operations560 // that might get inserted during vgpr regalloc.561 Register UnusedLowSGPR = TRI->findUnusedRegister(MRI, RC, MF);562 if (UnusedLowSGPR && TRI->getHWRegIndex(UnusedLowSGPR) <563 TRI->getHWRegIndex(FuncInfo->getSGPRForEXECCopy()))564 FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);565 } else {566 // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM567 // spills/copies. Reset the SGPR reserved for EXEC copy.568 FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);569 }570 571 SaveBlocks.clear();572 RestoreBlocks.clear();573 574 return MadeChange;575}576 577PreservedAnalyses578SILowerSGPRSpillsPass::run(MachineFunction &MF,579 MachineFunctionAnalysisManager &MFAM) {580 MFPropsModifier _(*this, MF);581 auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(MF);582 auto *Indexes = MFAM.getCachedResult<SlotIndexesAnalysis>(MF);583 MachineDominatorTree *MDT = &MFAM.getResult<MachineDominatorTreeAnalysis>(MF);584 SILowerSGPRSpills(LIS, Indexes, MDT).run(MF);585 return PreservedAnalyses::all();586}587