1111 lines · cpp
1//===- GCNRegPressure.cpp -------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file implements the GCNRegPressure class.11///12//===----------------------------------------------------------------------===//13 14#include "GCNRegPressure.h"15#include "AMDGPU.h"16#include "SIMachineFunctionInfo.h"17#include "llvm/CodeGen/MachineLoopInfo.h"18#include "llvm/CodeGen/RegisterPressure.h"19 20using namespace llvm;21 22#define DEBUG_TYPE "machine-scheduler"23 24bool llvm::isEqual(const GCNRPTracker::LiveRegSet &S1,25 const GCNRPTracker::LiveRegSet &S2) {26 if (S1.size() != S2.size())27 return false;28 29 for (const auto &P : S1) {30 auto I = S2.find(P.first);31 if (I == S2.end() || I->second != P.second)32 return false;33 }34 return true;35}36 37///////////////////////////////////////////////////////////////////////////////38// GCNRegPressure39 40unsigned GCNRegPressure::getRegKind(const TargetRegisterClass *RC,41 const SIRegisterInfo *STI) {42 return STI->isSGPRClass(RC)43 ? SGPR44 : (STI->isAGPRClass(RC)45 ? AGPR46 : (STI->isVectorSuperClass(RC) ? AVGPR : VGPR));47}48 49void GCNRegPressure::inc(unsigned Reg,50 LaneBitmask PrevMask,51 LaneBitmask NewMask,52 const MachineRegisterInfo &MRI) {53 unsigned NewNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(NewMask);54 unsigned PrevNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(PrevMask);55 if (NewNumCoveredRegs == PrevNumCoveredRegs)56 return;57 58 int Sign = 1;59 if (NewMask < PrevMask) {60 std::swap(NewMask, PrevMask);61 std::swap(NewNumCoveredRegs, PrevNumCoveredRegs);62 Sign = -1;63 }64 assert(PrevMask < NewMask && PrevNumCoveredRegs < NewNumCoveredRegs &&65 "prev mask should always be lesser than new");66 67 const TargetRegisterClass *RC = MRI.getRegClass(Reg);68 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();69 const SIRegisterInfo *STI = static_cast<const SIRegisterInfo *>(TRI);70 unsigned RegKind = getRegKind(RC, STI);71 if (TRI->getRegSizeInBits(*RC) != 32) {72 // Reg is from a tuple register class.73 if (PrevMask.none()) {74 unsigned TupleIdx = TOTAL_KINDS + RegKind;75 Value[TupleIdx] += Sign * TRI->getRegClassWeight(RC).RegWeight;76 }77 // Pressure scales with number of new registers covered by the new mask.78 // Note when true16 is enabled, we can no longer safely use the following79 // approach to calculate the difference in the number of 32-bit registers80 // between two masks:81 //82 // Sign *= SIRegisterInfo::getNumCoveredRegs(~PrevMask & NewMask);83 //84 // The issue is that the mask calculation `~PrevMask & NewMask` doesn't85 // properly account for partial usage of a 32-bit register when dealing with86 // 16-bit registers.87 //88 // Consider this example:89 // Assume PrevMask = 0b0010 and NewMask = 0b1111. Here, the correct register90 // usage difference should be 1, because even though PrevMask uses only half91 // of a 32-bit register, it should still be counted as a full register use.92 // However, the mask calculation yields `~PrevMask & NewMask = 0b1101`, and93 // calling `getNumCoveredRegs` returns 2 instead of 1. This incorrect94 // calculation can lead to integer overflow when Sign = -1.95 Sign *= NewNumCoveredRegs - PrevNumCoveredRegs;96 }97 Value[RegKind] += Sign;98}99 100bool GCNRegPressure::less(const MachineFunction &MF, const GCNRegPressure &O,101 unsigned MaxOccupancy) const {102 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();103 unsigned DynamicVGPRBlockSize =104 MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();105 106 const auto SGPROcc = std::min(MaxOccupancy,107 ST.getOccupancyWithNumSGPRs(getSGPRNum()));108 const auto VGPROcc = std::min(109 MaxOccupancy, ST.getOccupancyWithNumVGPRs(getVGPRNum(ST.hasGFX90AInsts()),110 DynamicVGPRBlockSize));111 const auto OtherSGPROcc = std::min(MaxOccupancy,112 ST.getOccupancyWithNumSGPRs(O.getSGPRNum()));113 const auto OtherVGPROcc =114 std::min(MaxOccupancy,115 ST.getOccupancyWithNumVGPRs(O.getVGPRNum(ST.hasGFX90AInsts()),116 DynamicVGPRBlockSize));117 118 const auto Occ = std::min(SGPROcc, VGPROcc);119 const auto OtherOcc = std::min(OtherSGPROcc, OtherVGPROcc);120 121 // Give first precedence to the better occupancy.122 if (Occ != OtherOcc)123 return Occ > OtherOcc;124 125 unsigned MaxVGPRs = ST.getMaxNumVGPRs(MF);126 unsigned MaxSGPRs = ST.getMaxNumSGPRs(MF);127 128 // SGPR excess pressure conditions129 unsigned ExcessSGPR = std::max(static_cast<int>(getSGPRNum() - MaxSGPRs), 0);130 unsigned OtherExcessSGPR =131 std::max(static_cast<int>(O.getSGPRNum() - MaxSGPRs), 0);132 133 auto WaveSize = ST.getWavefrontSize();134 // The number of virtual VGPRs required to handle excess SGPR135 unsigned VGPRForSGPRSpills = (ExcessSGPR + (WaveSize - 1)) / WaveSize;136 unsigned OtherVGPRForSGPRSpills =137 (OtherExcessSGPR + (WaveSize - 1)) / WaveSize;138 139 unsigned MaxArchVGPRs = ST.getAddressableNumArchVGPRs();140 141 // Unified excess pressure conditions, accounting for VGPRs used for SGPR142 // spills143 unsigned ExcessVGPR =144 std::max(static_cast<int>(getVGPRNum(ST.hasGFX90AInsts()) +145 VGPRForSGPRSpills - MaxVGPRs),146 0);147 unsigned OtherExcessVGPR =148 std::max(static_cast<int>(O.getVGPRNum(ST.hasGFX90AInsts()) +149 OtherVGPRForSGPRSpills - MaxVGPRs),150 0);151 // Arch VGPR excess pressure conditions, accounting for VGPRs used for SGPR152 // spills153 unsigned ExcessArchVGPR = std::max(154 static_cast<int>(getVGPRNum(false) + VGPRForSGPRSpills - MaxArchVGPRs),155 0);156 unsigned OtherExcessArchVGPR =157 std::max(static_cast<int>(O.getVGPRNum(false) + OtherVGPRForSGPRSpills -158 MaxArchVGPRs),159 0);160 // AGPR excess pressure conditions161 unsigned ExcessAGPR = std::max(162 static_cast<int>(ST.hasGFX90AInsts() ? (getAGPRNum() - MaxArchVGPRs)163 : (getAGPRNum() - MaxVGPRs)),164 0);165 unsigned OtherExcessAGPR = std::max(166 static_cast<int>(ST.hasGFX90AInsts() ? (O.getAGPRNum() - MaxArchVGPRs)167 : (O.getAGPRNum() - MaxVGPRs)),168 0);169 170 bool ExcessRP = ExcessSGPR || ExcessVGPR || ExcessArchVGPR || ExcessAGPR;171 bool OtherExcessRP = OtherExcessSGPR || OtherExcessVGPR ||172 OtherExcessArchVGPR || OtherExcessAGPR;173 174 // Give second precedence to the reduced number of spills to hold the register175 // pressure.176 if (ExcessRP || OtherExcessRP) {177 // The difference in excess VGPR pressure, after including VGPRs used for178 // SGPR spills179 int VGPRDiff = ((OtherExcessVGPR + OtherExcessArchVGPR + OtherExcessAGPR) -180 (ExcessVGPR + ExcessArchVGPR + ExcessAGPR));181 182 int SGPRDiff = OtherExcessSGPR - ExcessSGPR;183 184 if (VGPRDiff != 0)185 return VGPRDiff > 0;186 if (SGPRDiff != 0) {187 unsigned PureExcessVGPR =188 std::max(static_cast<int>(getVGPRNum(ST.hasGFX90AInsts()) - MaxVGPRs),189 0) +190 std::max(static_cast<int>(getVGPRNum(false) - MaxArchVGPRs), 0);191 unsigned OtherPureExcessVGPR =192 std::max(193 static_cast<int>(O.getVGPRNum(ST.hasGFX90AInsts()) - MaxVGPRs),194 0) +195 std::max(static_cast<int>(O.getVGPRNum(false) - MaxArchVGPRs), 0);196 197 // If we have a special case where there is a tie in excess VGPR, but one198 // of the pressures has VGPR usage from SGPR spills, prefer the pressure199 // with SGPR spills.200 if (PureExcessVGPR != OtherPureExcessVGPR)201 return SGPRDiff < 0;202 // If both pressures have the same excess pressure before and after203 // accounting for SGPR spills, prefer fewer SGPR spills.204 return SGPRDiff > 0;205 }206 }207 208 bool SGPRImportant = SGPROcc < VGPROcc;209 const bool OtherSGPRImportant = OtherSGPROcc < OtherVGPROcc;210 211 // If both pressures disagree on what is more important compare vgprs.212 if (SGPRImportant != OtherSGPRImportant) {213 SGPRImportant = false;214 }215 216 // Give third precedence to lower register tuple pressure.217 bool SGPRFirst = SGPRImportant;218 for (int I = 2; I > 0; --I, SGPRFirst = !SGPRFirst) {219 if (SGPRFirst) {220 auto SW = getSGPRTuplesWeight();221 auto OtherSW = O.getSGPRTuplesWeight();222 if (SW != OtherSW)223 return SW < OtherSW;224 } else {225 auto VW = getVGPRTuplesWeight();226 auto OtherVW = O.getVGPRTuplesWeight();227 if (VW != OtherVW)228 return VW < OtherVW;229 }230 }231 232 // Give final precedence to lower general RP.233 return SGPRImportant ? (getSGPRNum() < O.getSGPRNum()):234 (getVGPRNum(ST.hasGFX90AInsts()) <235 O.getVGPRNum(ST.hasGFX90AInsts()));236}237 238Printable llvm::print(const GCNRegPressure &RP, const GCNSubtarget *ST,239 unsigned DynamicVGPRBlockSize) {240 return Printable([&RP, ST, DynamicVGPRBlockSize](raw_ostream &OS) {241 OS << "VGPRs: " << RP.getArchVGPRNum() << ' '242 << "AGPRs: " << RP.getAGPRNum();243 if (ST)244 OS << "(O"245 << ST->getOccupancyWithNumVGPRs(RP.getVGPRNum(ST->hasGFX90AInsts()),246 DynamicVGPRBlockSize)247 << ')';248 OS << ", SGPRs: " << RP.getSGPRNum();249 if (ST)250 OS << "(O" << ST->getOccupancyWithNumSGPRs(RP.getSGPRNum()) << ')';251 OS << ", LVGPR WT: " << RP.getVGPRTuplesWeight()252 << ", LSGPR WT: " << RP.getSGPRTuplesWeight();253 if (ST)254 OS << " -> Occ: " << RP.getOccupancy(*ST, DynamicVGPRBlockSize);255 OS << '\n';256 });257}258 259static LaneBitmask getDefRegMask(const MachineOperand &MO,260 const MachineRegisterInfo &MRI) {261 assert(MO.isDef() && MO.isReg() && MO.getReg().isVirtual());262 263 // We don't rely on read-undef flag because in case of tentative schedule264 // tracking it isn't set correctly yet. This works correctly however since265 // use mask has been tracked before using LIS.266 return MO.getSubReg() == 0 ?267 MRI.getMaxLaneMaskForVReg(MO.getReg()) :268 MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(MO.getSubReg());269}270 271static void272collectVirtualRegUses(SmallVectorImpl<VRegMaskOrUnit> &VRegMaskOrUnits,273 const MachineInstr &MI, const LiveIntervals &LIS,274 const MachineRegisterInfo &MRI) {275 276 auto &TRI = *MRI.getTargetRegisterInfo();277 for (const auto &MO : MI.operands()) {278 if (!MO.isReg() || !MO.getReg().isVirtual())279 continue;280 if (!MO.isUse() || !MO.readsReg())281 continue;282 283 Register Reg = MO.getReg();284 auto I = llvm::find_if(VRegMaskOrUnits, [Reg](const VRegMaskOrUnit &RM) {285 return RM.VRegOrUnit.asVirtualReg() == Reg;286 });287 288 auto &P = I == VRegMaskOrUnits.end()289 ? VRegMaskOrUnits.emplace_back(VirtRegOrUnit(Reg),290 LaneBitmask::getNone())291 : *I;292 293 P.LaneMask |= MO.getSubReg() ? TRI.getSubRegIndexLaneMask(MO.getSubReg())294 : MRI.getMaxLaneMaskForVReg(Reg);295 }296 297 SlotIndex InstrSI;298 for (auto &P : VRegMaskOrUnits) {299 auto &LI = LIS.getInterval(P.VRegOrUnit.asVirtualReg());300 if (!LI.hasSubRanges())301 continue;302 303 // For a tentative schedule LIS isn't updated yet but livemask should304 // remain the same on any schedule. Subreg defs can be reordered but they305 // all must dominate uses anyway.306 if (!InstrSI)307 InstrSI = LIS.getInstructionIndex(MI).getBaseIndex();308 309 P.LaneMask = getLiveLaneMask(LI, InstrSI, MRI, P.LaneMask);310 }311}312 313/// Mostly copy/paste from CodeGen/RegisterPressure.cpp314static LaneBitmask getLanesWithProperty(315 const LiveIntervals &LIS, const MachineRegisterInfo &MRI,316 bool TrackLaneMasks, Register Reg, SlotIndex Pos,317 function_ref<bool(const LiveRange &LR, SlotIndex Pos)> Property) {318 assert(Reg.isVirtual());319 const LiveInterval &LI = LIS.getInterval(Reg);320 LaneBitmask Result;321 if (TrackLaneMasks && LI.hasSubRanges()) {322 for (const LiveInterval::SubRange &SR : LI.subranges()) {323 if (Property(SR, Pos))324 Result |= SR.LaneMask;325 }326 } else if (Property(LI, Pos)) {327 Result =328 TrackLaneMasks ? MRI.getMaxLaneMaskForVReg(Reg) : LaneBitmask::getAll();329 }330 331 return Result;332}333 334/// Mostly copy/paste from CodeGen/RegisterPressure.cpp335/// Helper to find a vreg use between two indices {PriorUseIdx, NextUseIdx}.336/// The query starts with a lane bitmask which gets lanes/bits removed for every337/// use we find.338static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,339 SlotIndex PriorUseIdx, SlotIndex NextUseIdx,340 const MachineRegisterInfo &MRI,341 const SIRegisterInfo *TRI,342 const LiveIntervals *LIS,343 bool Upward = false) {344 for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {345 if (MO.isUndef())346 continue;347 const MachineInstr *MI = MO.getParent();348 SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();349 bool InRange = Upward ? (InstSlot > PriorUseIdx && InstSlot <= NextUseIdx)350 : (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx);351 if (!InRange)352 continue;353 354 unsigned SubRegIdx = MO.getSubReg();355 LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(SubRegIdx);356 LastUseMask &= ~UseMask;357 if (LastUseMask.none())358 return LaneBitmask::getNone();359 }360 return LastUseMask;361}362 363////////////////////////////////////////////////////////////////////////////////364// GCNRPTarget365 366GCNRPTarget::GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP)367 : GCNRPTarget(RP, MF) {368 const Function &F = MF.getFunction();369 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();370 setTarget(ST.getMaxNumSGPRs(F), ST.getMaxNumVGPRs(F));371}372 373GCNRPTarget::GCNRPTarget(unsigned NumSGPRs, unsigned NumVGPRs,374 const MachineFunction &MF, const GCNRegPressure &RP)375 : GCNRPTarget(RP, MF) {376 setTarget(NumSGPRs, NumVGPRs);377}378 379GCNRPTarget::GCNRPTarget(unsigned Occupancy, const MachineFunction &MF,380 const GCNRegPressure &RP)381 : GCNRPTarget(RP, MF) {382 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();383 unsigned DynamicVGPRBlockSize =384 MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();385 setTarget(ST.getMaxNumSGPRs(Occupancy, /*Addressable=*/false),386 ST.getMaxNumVGPRs(Occupancy, DynamicVGPRBlockSize));387}388 389void GCNRPTarget::setTarget(unsigned NumSGPRs, unsigned NumVGPRs) {390 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();391 MaxSGPRs = std::min(ST.getAddressableNumSGPRs(), NumSGPRs);392 MaxVGPRs = std::min(ST.getAddressableNumArchVGPRs(), NumVGPRs);393 if (UnifiedRF) {394 unsigned DynamicVGPRBlockSize =395 MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();396 MaxUnifiedVGPRs =397 std::min(ST.getAddressableNumVGPRs(DynamicVGPRBlockSize), NumVGPRs);398 } else {399 MaxUnifiedVGPRs = 0;400 }401}402 403bool GCNRPTarget::isSaveBeneficial(Register Reg) const {404 const MachineRegisterInfo &MRI = MF.getRegInfo();405 const TargetRegisterClass *RC = MRI.getRegClass(Reg);406 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();407 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);408 409 if (SRI->isSGPRClass(RC))410 return RP.getSGPRNum() > MaxSGPRs;411 unsigned NumVGPRs =412 SRI->isAGPRClass(RC) ? RP.getAGPRNum() : RP.getArchVGPRNum();413 // The addressable limit must always be respected.414 if (NumVGPRs > MaxVGPRs)415 return true;416 // For unified RFs, combined VGPR usage limit must be respected as well.417 return UnifiedRF && RP.getVGPRNum(true) > MaxUnifiedVGPRs;418}419 420bool GCNRPTarget::satisfied() const {421 if (RP.getSGPRNum() > MaxSGPRs || RP.getVGPRNum(false) > MaxVGPRs)422 return false;423 if (UnifiedRF && RP.getVGPRNum(true) > MaxUnifiedVGPRs)424 return false;425 return true;426}427 428///////////////////////////////////////////////////////////////////////////////429// GCNRPTracker430 431LaneBitmask llvm::getLiveLaneMask(unsigned Reg, SlotIndex SI,432 const LiveIntervals &LIS,433 const MachineRegisterInfo &MRI,434 LaneBitmask LaneMaskFilter) {435 return getLiveLaneMask(LIS.getInterval(Reg), SI, MRI, LaneMaskFilter);436}437 438LaneBitmask llvm::getLiveLaneMask(const LiveInterval &LI, SlotIndex SI,439 const MachineRegisterInfo &MRI,440 LaneBitmask LaneMaskFilter) {441 LaneBitmask LiveMask;442 if (LI.hasSubRanges()) {443 for (const auto &S : LI.subranges())444 if ((S.LaneMask & LaneMaskFilter).any() && S.liveAt(SI)) {445 LiveMask |= S.LaneMask;446 assert(LiveMask == (LiveMask & MRI.getMaxLaneMaskForVReg(LI.reg())));447 }448 } else if (LI.liveAt(SI)) {449 LiveMask = MRI.getMaxLaneMaskForVReg(LI.reg());450 }451 LiveMask &= LaneMaskFilter;452 return LiveMask;453}454 455GCNRPTracker::LiveRegSet llvm::getLiveRegs(SlotIndex SI,456 const LiveIntervals &LIS,457 const MachineRegisterInfo &MRI,458 GCNRegPressure::RegKind RegKind) {459 GCNRPTracker::LiveRegSet LiveRegs;460 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {461 auto Reg = Register::index2VirtReg(I);462 if (RegKind != GCNRegPressure::TOTAL_KINDS &&463 GCNRegPressure::getRegKind(Reg, MRI) != RegKind)464 continue;465 if (!LIS.hasInterval(Reg))466 continue;467 auto LiveMask = getLiveLaneMask(Reg, SI, LIS, MRI);468 if (LiveMask.any())469 LiveRegs[Reg] = LiveMask;470 }471 return LiveRegs;472}473 474void GCNRPTracker::reset(const MachineInstr &MI,475 const LiveRegSet *LiveRegsCopy,476 bool After) {477 const MachineFunction &MF = *MI.getMF();478 MRI = &MF.getRegInfo();479 if (LiveRegsCopy) {480 if (&LiveRegs != LiveRegsCopy)481 LiveRegs = *LiveRegsCopy;482 } else {483 LiveRegs = After ? getLiveRegsAfter(MI, LIS)484 : getLiveRegsBefore(MI, LIS);485 }486 487 MaxPressure = CurPressure = getRegPressure(*MRI, LiveRegs);488}489 490void GCNRPTracker::reset(const MachineRegisterInfo &MRI_,491 const LiveRegSet &LiveRegs_) {492 MRI = &MRI_;493 LiveRegs = LiveRegs_;494 LastTrackedMI = nullptr;495 MaxPressure = CurPressure = getRegPressure(MRI_, LiveRegs_);496}497 498/// Mostly copy/paste from CodeGen/RegisterPressure.cpp499LaneBitmask GCNRPTracker::getLastUsedLanes(Register Reg, SlotIndex Pos) const {500 return getLanesWithProperty(501 LIS, *MRI, true, Reg, Pos.getBaseIndex(),502 [](const LiveRange &LR, SlotIndex Pos) {503 const LiveRange::Segment *S = LR.getSegmentContaining(Pos);504 return S != nullptr && S->end == Pos.getRegSlot();505 });506}507 508////////////////////////////////////////////////////////////////////////////////509// GCNUpwardRPTracker510 511void GCNUpwardRPTracker::recede(const MachineInstr &MI) {512 assert(MRI && "call reset first");513 514 LastTrackedMI = &MI;515 516 if (MI.isDebugInstr())517 return;518 519 // Kill all defs.520 GCNRegPressure DefPressure, ECDefPressure;521 bool HasECDefs = false;522 for (const MachineOperand &MO : MI.all_defs()) {523 if (!MO.getReg().isVirtual())524 continue;525 526 Register Reg = MO.getReg();527 LaneBitmask DefMask = getDefRegMask(MO, *MRI);528 529 // Treat a def as fully live at the moment of definition: keep a record.530 if (MO.isEarlyClobber()) {531 ECDefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);532 HasECDefs = true;533 } else534 DefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI);535 536 auto I = LiveRegs.find(Reg);537 if (I == LiveRegs.end())538 continue;539 540 LaneBitmask &LiveMask = I->second;541 LaneBitmask PrevMask = LiveMask;542 LiveMask &= ~DefMask;543 CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);544 if (LiveMask.none())545 LiveRegs.erase(I);546 }547 548 // Update MaxPressure with defs pressure.549 DefPressure += CurPressure;550 if (HasECDefs)551 DefPressure += ECDefPressure;552 MaxPressure = max(DefPressure, MaxPressure);553 554 // Make uses alive.555 SmallVector<VRegMaskOrUnit, 8> RegUses;556 collectVirtualRegUses(RegUses, MI, LIS, *MRI);557 for (const VRegMaskOrUnit &U : RegUses) {558 LaneBitmask &LiveMask = LiveRegs[U.VRegOrUnit.asVirtualReg()];559 LaneBitmask PrevMask = LiveMask;560 LiveMask |= U.LaneMask;561 CurPressure.inc(U.VRegOrUnit.asVirtualReg(), PrevMask, LiveMask, *MRI);562 }563 564 // Update MaxPressure with uses plus early-clobber defs pressure.565 MaxPressure = HasECDefs ? max(CurPressure + ECDefPressure, MaxPressure)566 : max(CurPressure, MaxPressure);567 568 assert(CurPressure == getRegPressure(*MRI, LiveRegs));569}570 571////////////////////////////////////////////////////////////////////////////////572// GCNDownwardRPTracker573 574bool GCNDownwardRPTracker::reset(const MachineInstr &MI,575 const LiveRegSet *LiveRegsCopy) {576 MRI = &MI.getMF()->getRegInfo();577 LastTrackedMI = nullptr;578 MBBEnd = MI.getParent()->end();579 NextMI = &MI;580 NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);581 if (NextMI == MBBEnd)582 return false;583 GCNRPTracker::reset(*NextMI, LiveRegsCopy, false);584 return true;585}586 587bool GCNDownwardRPTracker::advanceBeforeNext(MachineInstr *MI,588 bool UseInternalIterator) {589 assert(MRI && "call reset first");590 SlotIndex SI;591 const MachineInstr *CurrMI;592 if (UseInternalIterator) {593 if (!LastTrackedMI)594 return NextMI == MBBEnd;595 596 assert(NextMI == MBBEnd || !NextMI->isDebugInstr());597 CurrMI = LastTrackedMI;598 599 SI = NextMI == MBBEnd600 ? LIS.getInstructionIndex(*LastTrackedMI).getDeadSlot()601 : LIS.getInstructionIndex(*NextMI).getBaseIndex();602 } else { //! UseInternalIterator603 SI = LIS.getInstructionIndex(*MI).getBaseIndex();604 CurrMI = MI;605 }606 607 assert(SI.isValid());608 609 // Remove dead registers or mask bits.610 SmallSet<Register, 8> SeenRegs;611 for (auto &MO : CurrMI->operands()) {612 if (!MO.isReg() || !MO.getReg().isVirtual())613 continue;614 if (MO.isUse() && !MO.readsReg())615 continue;616 if (!UseInternalIterator && MO.isDef())617 continue;618 if (!SeenRegs.insert(MO.getReg()).second)619 continue;620 const LiveInterval &LI = LIS.getInterval(MO.getReg());621 if (LI.hasSubRanges()) {622 auto It = LiveRegs.end();623 for (const auto &S : LI.subranges()) {624 if (!S.liveAt(SI)) {625 if (It == LiveRegs.end()) {626 It = LiveRegs.find(MO.getReg());627 if (It == LiveRegs.end())628 llvm_unreachable("register isn't live");629 }630 auto PrevMask = It->second;631 It->second &= ~S.LaneMask;632 CurPressure.inc(MO.getReg(), PrevMask, It->second, *MRI);633 }634 }635 if (It != LiveRegs.end() && It->second.none())636 LiveRegs.erase(It);637 } else if (!LI.liveAt(SI)) {638 auto It = LiveRegs.find(MO.getReg());639 if (It == LiveRegs.end())640 llvm_unreachable("register isn't live");641 CurPressure.inc(MO.getReg(), It->second, LaneBitmask::getNone(), *MRI);642 LiveRegs.erase(It);643 }644 }645 646 MaxPressure = max(MaxPressure, CurPressure);647 648 LastTrackedMI = nullptr;649 650 return UseInternalIterator && (NextMI == MBBEnd);651}652 653void GCNDownwardRPTracker::advanceToNext(MachineInstr *MI,654 bool UseInternalIterator) {655 if (UseInternalIterator) {656 LastTrackedMI = &*NextMI++;657 NextMI = skipDebugInstructionsForward(NextMI, MBBEnd);658 } else {659 LastTrackedMI = MI;660 }661 662 const MachineInstr *CurrMI = LastTrackedMI;663 664 // Add new registers or mask bits.665 for (const auto &MO : CurrMI->all_defs()) {666 Register Reg = MO.getReg();667 if (!Reg.isVirtual())668 continue;669 auto &LiveMask = LiveRegs[Reg];670 auto PrevMask = LiveMask;671 LiveMask |= getDefRegMask(MO, *MRI);672 CurPressure.inc(Reg, PrevMask, LiveMask, *MRI);673 }674 675 MaxPressure = max(MaxPressure, CurPressure);676}677 678bool GCNDownwardRPTracker::advance(MachineInstr *MI, bool UseInternalIterator) {679 if (UseInternalIterator && NextMI == MBBEnd)680 return false;681 682 advanceBeforeNext(MI, UseInternalIterator);683 advanceToNext(MI, UseInternalIterator);684 if (!UseInternalIterator) {685 // We must remove any dead def lanes from the current RP686 advanceBeforeNext(MI, true);687 }688 return true;689}690 691bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator End) {692 while (NextMI != End)693 if (!advance()) return false;694 return true;695}696 697bool GCNDownwardRPTracker::advance(MachineBasicBlock::const_iterator Begin,698 MachineBasicBlock::const_iterator End,699 const LiveRegSet *LiveRegsCopy) {700 reset(*Begin, LiveRegsCopy);701 return advance(End);702}703 704Printable llvm::reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,705 const GCNRPTracker::LiveRegSet &TrackedLR,706 const TargetRegisterInfo *TRI, StringRef Pfx) {707 return Printable([&LISLR, &TrackedLR, TRI, Pfx](raw_ostream &OS) {708 for (auto const &P : TrackedLR) {709 auto I = LISLR.find(P.first);710 if (I == LISLR.end()) {711 OS << Pfx << printReg(P.first, TRI) << ":L" << PrintLaneMask(P.second)712 << " isn't found in LIS reported set\n";713 } else if (I->second != P.second) {714 OS << Pfx << printReg(P.first, TRI)715 << " masks doesn't match: LIS reported " << PrintLaneMask(I->second)716 << ", tracked " << PrintLaneMask(P.second) << '\n';717 }718 }719 for (auto const &P : LISLR) {720 auto I = TrackedLR.find(P.first);721 if (I == TrackedLR.end()) {722 OS << Pfx << printReg(P.first, TRI) << ":L" << PrintLaneMask(P.second)723 << " isn't found in tracked set\n";724 }725 }726 });727}728 729GCNRegPressure730GCNDownwardRPTracker::bumpDownwardPressure(const MachineInstr *MI,731 const SIRegisterInfo *TRI) const {732 assert(!MI->isDebugOrPseudoInstr() && "Expect a nondebug instruction.");733 734 SlotIndex SlotIdx;735 SlotIdx = LIS.getInstructionIndex(*MI).getRegSlot();736 737 // Account for register pressure similar to RegPressureTracker::recede().738 RegisterOperands RegOpers;739 RegOpers.collect(*MI, *TRI, *MRI, true, /*IgnoreDead=*/false);740 RegOpers.adjustLaneLiveness(LIS, *MRI, SlotIdx);741 GCNRegPressure TempPressure = CurPressure;742 743 for (const VRegMaskOrUnit &Use : RegOpers.Uses) {744 if (!Use.VRegOrUnit.isVirtualReg())745 continue;746 Register Reg = Use.VRegOrUnit.asVirtualReg();747 LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);748 if (LastUseMask.none())749 continue;750 // The LastUseMask is queried from the liveness information of instruction751 // which may be further down the schedule. Some lanes may actually not be752 // last uses for the current position.753 // FIXME: allow the caller to pass in the list of vreg uses that remain754 // to be bottom-scheduled to avoid searching uses at each query.755 SlotIndex CurrIdx;756 const MachineBasicBlock *MBB = MI->getParent();757 MachineBasicBlock::const_iterator IdxPos = skipDebugInstructionsForward(758 LastTrackedMI ? LastTrackedMI : MBB->begin(), MBB->end());759 if (IdxPos == MBB->end()) {760 CurrIdx = LIS.getMBBEndIdx(MBB);761 } else {762 CurrIdx = LIS.getInstructionIndex(*IdxPos).getRegSlot();763 }764 765 LastUseMask =766 findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, TRI, &LIS);767 if (LastUseMask.none())768 continue;769 770 auto It = LiveRegs.find(Reg);771 LaneBitmask LiveMask = It != LiveRegs.end() ? It->second : LaneBitmask(0);772 LaneBitmask NewMask = LiveMask & ~LastUseMask;773 TempPressure.inc(Reg, LiveMask, NewMask, *MRI);774 }775 776 // Generate liveness for defs.777 for (const VRegMaskOrUnit &Def : RegOpers.Defs) {778 if (!Def.VRegOrUnit.isVirtualReg())779 continue;780 Register Reg = Def.VRegOrUnit.asVirtualReg();781 auto It = LiveRegs.find(Reg);782 LaneBitmask LiveMask = It != LiveRegs.end() ? It->second : LaneBitmask(0);783 LaneBitmask NewMask = LiveMask | Def.LaneMask;784 TempPressure.inc(Reg, LiveMask, NewMask, *MRI);785 }786 787 return TempPressure;788}789 790bool GCNUpwardRPTracker::isValid() const {791 const auto &SI = LIS.getInstructionIndex(*LastTrackedMI).getBaseIndex();792 const auto LISLR = llvm::getLiveRegs(SI, LIS, *MRI);793 const auto &TrackedLR = LiveRegs;794 795 if (!isEqual(LISLR, TrackedLR)) {796 dbgs() << "\nGCNUpwardRPTracker error: Tracked and"797 " LIS reported livesets mismatch:\n"798 << print(LISLR, *MRI);799 reportMismatch(LISLR, TrackedLR, MRI->getTargetRegisterInfo());800 return false;801 }802 803 auto LISPressure = getRegPressure(*MRI, LISLR);804 if (LISPressure != CurPressure) {805 dbgs() << "GCNUpwardRPTracker error: Pressure sets different\nTracked: "806 << print(CurPressure) << "LIS rpt: " << print(LISPressure);807 return false;808 }809 return true;810}811 812Printable llvm::print(const GCNRPTracker::LiveRegSet &LiveRegs,813 const MachineRegisterInfo &MRI) {814 return Printable([&LiveRegs, &MRI](raw_ostream &OS) {815 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();816 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {817 Register Reg = Register::index2VirtReg(I);818 auto It = LiveRegs.find(Reg);819 if (It != LiveRegs.end() && It->second.any())820 OS << ' ' << printReg(Reg, TRI) << ':' << PrintLaneMask(It->second);821 }822 OS << '\n';823 });824}825 826void GCNRegPressure::dump() const { dbgs() << print(*this); }827 828static cl::opt<bool> UseDownwardTracker(829 "amdgpu-print-rp-downward",830 cl::desc("Use GCNDownwardRPTracker for GCNRegPressurePrinter pass"),831 cl::init(false), cl::Hidden);832 833char llvm::GCNRegPressurePrinter::ID = 0;834char &llvm::GCNRegPressurePrinterID = GCNRegPressurePrinter::ID;835 836INITIALIZE_PASS(GCNRegPressurePrinter, "amdgpu-print-rp", "", true, true)837 838// Return lanemask of Reg's subregs that are live-through at [Begin, End] and839// are fully covered by Mask.840static LaneBitmask841getRegLiveThroughMask(const MachineRegisterInfo &MRI, const LiveIntervals &LIS,842 Register Reg, SlotIndex Begin, SlotIndex End,843 LaneBitmask Mask = LaneBitmask::getAll()) {844 845 auto IsInOneSegment = [Begin, End](const LiveRange &LR) -> bool {846 auto *Segment = LR.getSegmentContaining(Begin);847 return Segment && Segment->contains(End);848 };849 850 LaneBitmask LiveThroughMask;851 const LiveInterval &LI = LIS.getInterval(Reg);852 if (LI.hasSubRanges()) {853 for (auto &SR : LI.subranges()) {854 if ((SR.LaneMask & Mask) == SR.LaneMask && IsInOneSegment(SR))855 LiveThroughMask |= SR.LaneMask;856 }857 } else {858 LaneBitmask RegMask = MRI.getMaxLaneMaskForVReg(Reg);859 if ((RegMask & Mask) == RegMask && IsInOneSegment(LI))860 LiveThroughMask = RegMask;861 }862 863 return LiveThroughMask;864}865 866bool GCNRegPressurePrinter::runOnMachineFunction(MachineFunction &MF) {867 const MachineRegisterInfo &MRI = MF.getRegInfo();868 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();869 const LiveIntervals &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();870 871 auto &OS = dbgs();872 873// Leading spaces are important for YAML syntax.874#define PFX " "875 876 OS << "---\nname: " << MF.getName() << "\nbody: |\n";877 878 auto printRP = [](const GCNRegPressure &RP) {879 return Printable([&RP](raw_ostream &OS) {880 OS << format(PFX " %-5d", RP.getSGPRNum())881 << format(" %-5d", RP.getVGPRNum(false));882 });883 };884 885 auto ReportLISMismatchIfAny = [&](const GCNRPTracker::LiveRegSet &TrackedLR,886 const GCNRPTracker::LiveRegSet &LISLR) {887 if (LISLR != TrackedLR) {888 OS << PFX " mis LIS: " << llvm::print(LISLR, MRI)889 << reportMismatch(LISLR, TrackedLR, TRI, PFX " ");890 }891 };892 893 // Register pressure before and at an instruction (in program order).894 SmallVector<std::pair<GCNRegPressure, GCNRegPressure>, 16> RP;895 896 for (auto &MBB : MF) {897 RP.clear();898 RP.reserve(MBB.size());899 900 OS << PFX;901 MBB.printName(OS);902 OS << ":\n";903 904 SlotIndex MBBStartSlot = LIS.getSlotIndexes()->getMBBStartIdx(&MBB);905 SlotIndex MBBLastSlot = LIS.getSlotIndexes()->getMBBLastIdx(&MBB);906 907 GCNRPTracker::LiveRegSet LiveIn, LiveOut;908 GCNRegPressure RPAtMBBEnd;909 910 if (UseDownwardTracker) {911 if (MBB.empty()) {912 LiveIn = LiveOut = getLiveRegs(MBBStartSlot, LIS, MRI);913 RPAtMBBEnd = getRegPressure(MRI, LiveIn);914 } else {915 GCNDownwardRPTracker RPT(LIS);916 RPT.reset(MBB.front());917 918 LiveIn = RPT.getLiveRegs();919 920 while (!RPT.advanceBeforeNext()) {921 GCNRegPressure RPBeforeMI = RPT.getPressure();922 RPT.advanceToNext();923 RP.emplace_back(RPBeforeMI, RPT.getPressure());924 }925 926 LiveOut = RPT.getLiveRegs();927 RPAtMBBEnd = RPT.getPressure();928 }929 } else {930 GCNUpwardRPTracker RPT(LIS);931 RPT.reset(MRI, MBBLastSlot);932 933 LiveOut = RPT.getLiveRegs();934 RPAtMBBEnd = RPT.getPressure();935 936 for (auto &MI : reverse(MBB)) {937 RPT.resetMaxPressure();938 RPT.recede(MI);939 if (!MI.isDebugInstr())940 RP.emplace_back(RPT.getPressure(), RPT.getMaxPressure());941 }942 943 LiveIn = RPT.getLiveRegs();944 }945 946 OS << PFX " Live-in: " << llvm::print(LiveIn, MRI);947 if (!UseDownwardTracker)948 ReportLISMismatchIfAny(LiveIn, getLiveRegs(MBBStartSlot, LIS, MRI));949 950 OS << PFX " SGPR VGPR\n";951 int I = 0;952 for (auto &MI : MBB) {953 if (!MI.isDebugInstr()) {954 auto &[RPBeforeInstr, RPAtInstr] =955 RP[UseDownwardTracker ? I : (RP.size() - 1 - I)];956 ++I;957 OS << printRP(RPBeforeInstr) << '\n' << printRP(RPAtInstr) << " ";958 } else959 OS << PFX " ";960 MI.print(OS);961 }962 OS << printRP(RPAtMBBEnd) << '\n';963 964 OS << PFX " Live-out:" << llvm::print(LiveOut, MRI);965 if (UseDownwardTracker)966 ReportLISMismatchIfAny(LiveOut, getLiveRegs(MBBLastSlot, LIS, MRI));967 968 GCNRPTracker::LiveRegSet LiveThrough;969 for (auto [Reg, Mask] : LiveIn) {970 LaneBitmask MaskIntersection = Mask & LiveOut.lookup(Reg);971 if (MaskIntersection.any()) {972 LaneBitmask LTMask = getRegLiveThroughMask(973 MRI, LIS, Reg, MBBStartSlot, MBBLastSlot, MaskIntersection);974 if (LTMask.any())975 LiveThrough[Reg] = LTMask;976 }977 }978 OS << PFX " Live-thr:" << llvm::print(LiveThrough, MRI);979 OS << printRP(getRegPressure(MRI, LiveThrough)) << '\n';980 }981 OS << "...\n";982 return false;983 984#undef PFX985}986 987#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)988LLVM_DUMP_METHOD void llvm::dumpMaxRegPressure(MachineFunction &MF,989 GCNRegPressure::RegKind Kind,990 LiveIntervals &LIS,991 const MachineLoopInfo *MLI) {992 993 const MachineRegisterInfo &MRI = MF.getRegInfo();994 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();995 auto &OS = dbgs();996 const char *RegName = GCNRegPressure::getName(Kind);997 998 unsigned MaxNumRegs = 0;999 const MachineInstr *MaxPressureMI = nullptr;1000 GCNUpwardRPTracker RPT(LIS);1001 for (const MachineBasicBlock &MBB : MF) {1002 RPT.reset(MRI, LIS.getSlotIndexes()->getMBBEndIdx(&MBB).getPrevSlot());1003 for (const MachineInstr &MI : reverse(MBB)) {1004 RPT.recede(MI);1005 unsigned NumRegs = RPT.getMaxPressure().getNumRegs(Kind);1006 if (NumRegs > MaxNumRegs) {1007 MaxNumRegs = NumRegs;1008 MaxPressureMI = &MI;1009 }1010 }1011 }1012 1013 SlotIndex MISlot = LIS.getInstructionIndex(*MaxPressureMI);1014 1015 // Max pressure can occur at either the early-clobber or register slot.1016 // Choose the maximum liveset between both slots. This is ugly but this is1017 // diagnostic code.1018 SlotIndex ECSlot = MISlot.getRegSlot(true);1019 SlotIndex RSlot = MISlot.getRegSlot(false);1020 GCNRPTracker::LiveRegSet ECLiveSet = getLiveRegs(ECSlot, LIS, MRI, Kind);1021 GCNRPTracker::LiveRegSet RLiveSet = getLiveRegs(RSlot, LIS, MRI, Kind);1022 unsigned ECNumRegs = getRegPressure(MRI, ECLiveSet).getNumRegs(Kind);1023 unsigned RNumRegs = getRegPressure(MRI, RLiveSet).getNumRegs(Kind);1024 GCNRPTracker::LiveRegSet *LiveSet =1025 ECNumRegs > RNumRegs ? &ECLiveSet : &RLiveSet;1026 SlotIndex MaxPressureSlot = ECNumRegs > RNumRegs ? ECSlot : RSlot;1027 assert(getRegPressure(MRI, *LiveSet).getNumRegs(Kind) == MaxNumRegs);1028 1029 // Split live registers into single-def and multi-def sets.1030 GCNRegPressure SDefPressure, MDefPressure;1031 SmallVector<Register, 16> SDefRegs, MDefRegs;1032 for (auto [Reg, LaneMask] : *LiveSet) {1033 assert(GCNRegPressure::getRegKind(Reg, MRI) == Kind);1034 LiveInterval &LI = LIS.getInterval(Reg);1035 if (LI.getNumValNums() == 1 ||1036 (LI.hasSubRanges() &&1037 llvm::all_of(LI.subranges(), [](const LiveInterval::SubRange &SR) {1038 return SR.getNumValNums() == 1;1039 }))) {1040 SDefPressure.inc(Reg, LaneBitmask::getNone(), LaneMask, MRI);1041 SDefRegs.push_back(Reg);1042 } else {1043 MDefPressure.inc(Reg, LaneBitmask::getNone(), LaneMask, MRI);1044 MDefRegs.push_back(Reg);1045 }1046 }1047 unsigned SDefNumRegs = SDefPressure.getNumRegs(Kind);1048 unsigned MDefNumRegs = MDefPressure.getNumRegs(Kind);1049 assert(SDefNumRegs + MDefNumRegs == MaxNumRegs);1050 1051 auto printLoc = [&](const MachineBasicBlock *MBB, SlotIndex SI) {1052 return Printable([&, MBB, SI](raw_ostream &OS) {1053 OS << SI << ':' << printMBBReference(*MBB);1054 if (MLI)1055 if (const MachineLoop *ML = MLI->getLoopFor(MBB))1056 OS << " (LoopHdr " << printMBBReference(*ML->getHeader())1057 << ", Depth " << ML->getLoopDepth() << ")";1058 });1059 };1060 1061 auto PrintRegInfo = [&](Register Reg, LaneBitmask LiveMask) {1062 GCNRegPressure RegPressure;1063 RegPressure.inc(Reg, LaneBitmask::getNone(), LiveMask, MRI);1064 OS << " " << printReg(Reg, TRI) << ':'1065 << TRI->getRegClassName(MRI.getRegClass(Reg)) << ", LiveMask "1066 << PrintLaneMask(LiveMask) << " (" << RegPressure.getNumRegs(Kind) << ' '1067 << RegName << "s)\n";1068 1069 // Use std::map to sort def/uses by SlotIndex.1070 std::map<SlotIndex, const MachineInstr *> Instrs;1071 for (const MachineInstr &MI : MRI.reg_nodbg_instructions(Reg)) {1072 Instrs[LIS.getInstructionIndex(MI).getRegSlot()] = &MI;1073 }1074 1075 for (const auto &[SI, MI] : Instrs) {1076 OS << " ";1077 if (MI->definesRegister(Reg, TRI))1078 OS << "def ";1079 if (MI->readsRegister(Reg, TRI))1080 OS << "use ";1081 OS << printLoc(MI->getParent(), SI) << ": " << *MI;1082 }1083 };1084 1085 OS << "\n*** Register pressure info (" << RegName << "s) for " << MF.getName()1086 << " ***\n";1087 OS << "Max pressure is " << MaxNumRegs << ' ' << RegName << "s at "1088 << printLoc(MaxPressureMI->getParent(), MaxPressureSlot) << ": "1089 << *MaxPressureMI;1090 1091 OS << "\nLive registers with single definition (" << SDefNumRegs << ' '1092 << RegName << "s):\n";1093 1094 // Sort SDefRegs by number of uses (smallest first)1095 llvm::sort(SDefRegs, [&](Register A, Register B) {1096 return std::distance(MRI.use_nodbg_begin(A), MRI.use_nodbg_end()) <1097 std::distance(MRI.use_nodbg_begin(B), MRI.use_nodbg_end());1098 });1099 1100 for (const Register Reg : SDefRegs) {1101 PrintRegInfo(Reg, LiveSet->lookup(Reg));1102 }1103 1104 OS << "\nLive registers with multiple definitions (" << MDefNumRegs << ' '1105 << RegName << "s):\n";1106 for (const Register Reg : MDefRegs) {1107 PrintRegInfo(Reg, LiveSet->lookup(Reg));1108 }1109}1110#endif1111