550 lines · c
1//===- GCNRegPressure.h -----------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file10/// This file defines the GCNRegPressure class, which tracks registry pressure11/// by bookkeeping number of SGPR/VGPRs used, weights for large SGPR/VGPRs. It12/// also implements a compare function, which compares different register13/// pressures, and declares one with max occupancy as winner.14///15//===----------------------------------------------------------------------===//16 17#ifndef LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H18#define LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H19 20#include "GCNSubtarget.h"21#include "llvm/CodeGen/LiveIntervals.h"22#include "llvm/CodeGen/RegisterPressure.h"23#include <algorithm>24#include <array>25 26namespace llvm {27 28class MachineRegisterInfo;29class raw_ostream;30class SlotIndex;31 32struct GCNRegPressure {33 enum RegKind { SGPR, VGPR, AGPR, AVGPR, TOTAL_KINDS };34 35 static constexpr const char *getName(RegKind Kind) {36 const char *Names[] = {"SGPR", "VGPR", "AGPR", "AVGPR"};37 assert(Kind < TOTAL_KINDS);38 return Names[Kind];39 }40 41 GCNRegPressure() {42 clear();43 }44 45 bool empty() const {46 return !Value[SGPR] && !Value[VGPR] && !Value[AGPR] && !Value[AVGPR];47 }48 49 void clear() { Value.fill(0); }50 51 unsigned getNumRegs(RegKind Kind) const {52 assert(Kind < TOTAL_KINDS);53 return Value[Kind];54 }55 56 /// \returns the SGPR32 pressure57 unsigned getSGPRNum() const { return Value[SGPR]; }58 /// \returns the aggregated ArchVGPR32, AccVGPR32, and Pseudo AVGPR pressure59 /// dependent upon \p UnifiedVGPRFile60 unsigned getVGPRNum(bool UnifiedVGPRFile) const {61 if (UnifiedVGPRFile) {62 return Value[AGPR]63 ? getUnifiedVGPRNum(Value[VGPR], Value[AGPR], Value[AVGPR])64 : Value[VGPR] + Value[AVGPR];65 }66 // AVGPR assignment priority is based on the width of the register. Account67 // AVGPR pressure as VGPR.68 return std::max(Value[VGPR] + Value[AVGPR], Value[AGPR]);69 }70 71 /// Returns the aggregated VGPR pressure, assuming \p NumArchVGPRs ArchVGPRs72 /// \p NumAGPRs AGPRS, and \p NumAVGPRs AVGPRs for a target with a unified73 /// VGPR file.74 inline static unsigned getUnifiedVGPRNum(unsigned NumArchVGPRs,75 unsigned NumAGPRs,76 unsigned NumAVGPRs) {77 78 // Assume AVGPRs will be assigned as VGPRs.79 return alignTo(NumArchVGPRs + NumAVGPRs,80 AMDGPU::IsaInfo::getArchVGPRAllocGranule()) +81 NumAGPRs;82 }83 84 /// \returns the ArchVGPR32 pressure, plus the AVGPRS which we assume will be85 /// allocated as VGPR86 unsigned getArchVGPRNum() const { return Value[VGPR] + Value[AVGPR]; }87 /// \returns the AccVGPR32 pressure88 unsigned getAGPRNum() const { return Value[AGPR]; }89 /// \returns the AVGPR32 pressure90 unsigned getAVGPRNum() const { return Value[AVGPR]; }91 92 unsigned getVGPRTuplesWeight() const {93 return std::max(Value[TOTAL_KINDS + VGPR] + Value[TOTAL_KINDS + AVGPR],94 Value[TOTAL_KINDS + AGPR]);95 }96 unsigned getSGPRTuplesWeight() const { return Value[TOTAL_KINDS + SGPR]; }97 98 unsigned getOccupancy(const GCNSubtarget &ST,99 unsigned DynamicVGPRBlockSize) const {100 return std::min(ST.getOccupancyWithNumSGPRs(getSGPRNum()),101 ST.getOccupancyWithNumVGPRs(getVGPRNum(ST.hasGFX90AInsts()),102 DynamicVGPRBlockSize));103 }104 105 void inc(unsigned Reg,106 LaneBitmask PrevMask,107 LaneBitmask NewMask,108 const MachineRegisterInfo &MRI);109 110 bool higherOccupancy(const GCNSubtarget &ST, const GCNRegPressure &O,111 unsigned DynamicVGPRBlockSize) const {112 return getOccupancy(ST, DynamicVGPRBlockSize) >113 O.getOccupancy(ST, DynamicVGPRBlockSize);114 }115 116 /// Compares \p this GCNRegpressure to \p O, returning true if \p this is117 /// less. Since GCNRegpressure contains different types of pressures, and due118 /// to target-specific pecularities (e.g. we care about occupancy rather than119 /// raw register usage), we determine if \p this GCNRegPressure is less than120 /// \p O based on the following tiered comparisons (in order order of121 /// precedence):122 /// 1. Better occupancy123 /// 2. Less spilling (first preference to VGPR spills, then to SGPR spills)124 /// 3. Less tuple register pressure (first preference to VGPR tuples if we125 /// determine that SGPR pressure is not important)126 /// 4. Less raw register pressure (first preference to VGPR tuples if we127 /// determine that SGPR pressure is not important)128 bool less(const MachineFunction &MF, const GCNRegPressure &O,129 unsigned MaxOccupancy = std::numeric_limits<unsigned>::max()) const;130 131 bool operator==(const GCNRegPressure &O) const { return Value == O.Value; }132 133 bool operator!=(const GCNRegPressure &O) const {134 return !(*this == O);135 }136 137 GCNRegPressure &operator+=(const GCNRegPressure &RHS) {138 for (unsigned I = 0; I < ValueArraySize; ++I)139 Value[I] += RHS.Value[I];140 return *this;141 }142 143 GCNRegPressure &operator-=(const GCNRegPressure &RHS) {144 for (unsigned I = 0; I < ValueArraySize; ++I)145 Value[I] -= RHS.Value[I];146 return *this;147 }148 149 void dump() const;150 151 static RegKind getRegKind(unsigned Reg, const MachineRegisterInfo &MRI) {152 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();153 const SIRegisterInfo *STI = static_cast<const SIRegisterInfo *>(TRI);154 return (RegKind)getRegKind(MRI.getRegClass(Reg), STI);155 }156 157private:158 static constexpr unsigned ValueArraySize = TOTAL_KINDS * 2;159 160 /// Pressure for all register kinds (first all regular registers kinds, then161 /// all tuple register kinds).162 std::array<unsigned, ValueArraySize> Value;163 164 static unsigned getRegKind(const TargetRegisterClass *RC,165 const SIRegisterInfo *STI);166 167 friend GCNRegPressure max(const GCNRegPressure &P1,168 const GCNRegPressure &P2);169 170 friend Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST,171 unsigned DynamicVGPRBlockSize);172};173 174inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) {175 GCNRegPressure Res;176 for (unsigned I = 0; I < GCNRegPressure::ValueArraySize; ++I)177 Res.Value[I] = std::max(P1.Value[I], P2.Value[I]);178 return Res;179}180 181inline GCNRegPressure operator+(const GCNRegPressure &P1,182 const GCNRegPressure &P2) {183 GCNRegPressure Sum = P1;184 Sum += P2;185 return Sum;186}187 188inline GCNRegPressure operator-(const GCNRegPressure &P1,189 const GCNRegPressure &P2) {190 GCNRegPressure Diff = P1;191 Diff -= P2;192 return Diff;193}194 195////////////////////////////////////////////////////////////////////////////////196// GCNRPTarget197 198/// Models a register pressure target, allowing to evaluate and track register199/// savings against that target from a starting \ref GCNRegPressure.200class GCNRPTarget {201public:202 /// Sets up the target such that the register pressure starting at \p RP does203 /// not show register spilling on function \p MF (w.r.t. the function's204 /// mininum target occupancy).205 GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP);206 207 /// Sets up the target such that the register pressure starting at \p RP does208 /// not use more than \p NumSGPRs SGPRs and \p NumVGPRs VGPRs on function \p209 /// MF.210 GCNRPTarget(unsigned NumSGPRs, unsigned NumVGPRs, const MachineFunction &MF,211 const GCNRegPressure &RP);212 213 /// Sets up the target such that the register pressure starting at \p RP does214 /// not prevent achieving an occupancy of at least \p Occupancy on function215 /// \p MF.216 GCNRPTarget(unsigned Occupancy, const MachineFunction &MF,217 const GCNRegPressure &RP);218 219 /// Changes the target (same semantics as constructor).220 void setTarget(unsigned NumSGPRs, unsigned NumVGPRs);221 222 const GCNRegPressure &getCurrentRP() const { return RP; }223 224 void setRP(const GCNRegPressure &NewRP) { RP = NewRP; }225 226 /// Determines whether saving virtual register \p Reg will be beneficial227 /// towards achieving the RP target.228 bool isSaveBeneficial(Register Reg) const;229 230 /// Saves virtual register \p Reg with lanemask \p Mask.231 void saveReg(Register Reg, LaneBitmask Mask, const MachineRegisterInfo &MRI) {232 RP.inc(Reg, Mask, LaneBitmask::getNone(), MRI);233 }234 235 /// Whether the current RP is at or below the defined pressure target.236 bool satisfied() const;237 238#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)239 friend raw_ostream &operator<<(raw_ostream &OS, const GCNRPTarget &Target) {240 OS << "Actual/Target: " << Target.RP.getSGPRNum() << '/' << Target.MaxSGPRs241 << " SGPRs, " << Target.RP.getArchVGPRNum() << '/' << Target.MaxVGPRs242 << " ArchVGPRs, " << Target.RP.getAGPRNum() << '/' << Target.MaxVGPRs243 << " AGPRs";244 245 if (Target.MaxUnifiedVGPRs) {246 OS << ", " << Target.RP.getVGPRNum(true) << '/' << Target.MaxUnifiedVGPRs247 << " VGPRs (unified)";248 }249 return OS;250 }251#endif252 253private:254 const MachineFunction &MF;255 const bool UnifiedRF;256 257 /// Current register pressure.258 GCNRegPressure RP;259 260 /// Target number of SGPRs.261 unsigned MaxSGPRs;262 /// Target number of ArchVGPRs and AGPRs.263 unsigned MaxVGPRs;264 /// Target number of overall VGPRs for subtargets with unified RFs. Always 0265 /// for subtargets with non-unified RFs.266 unsigned MaxUnifiedVGPRs;267 268 GCNRPTarget(const GCNRegPressure &RP, const MachineFunction &MF)269 : MF(MF), UnifiedRF(MF.getSubtarget<GCNSubtarget>().hasGFX90AInsts()),270 RP(RP) {}271};272 273///////////////////////////////////////////////////////////////////////////////274// GCNRPTracker275 276class GCNRPTracker {277public:278 using LiveRegSet = DenseMap<unsigned, LaneBitmask>;279 280protected:281 const LiveIntervals &LIS;282 LiveRegSet LiveRegs;283 GCNRegPressure CurPressure, MaxPressure;284 const MachineInstr *LastTrackedMI = nullptr;285 mutable const MachineRegisterInfo *MRI = nullptr;286 287 GCNRPTracker(const LiveIntervals &LIS_) : LIS(LIS_) {}288 289 void reset(const MachineInstr &MI, const LiveRegSet *LiveRegsCopy,290 bool After);291 292 /// Mostly copy/paste from CodeGen/RegisterPressure.cpp293 void bumpDeadDefs(ArrayRef<VRegMaskOrUnit> DeadDefs);294 295 LaneBitmask getLastUsedLanes(Register Reg, SlotIndex Pos) const;296 297public:298 // reset tracker and set live register set to the specified value.299 void reset(const MachineRegisterInfo &MRI_, const LiveRegSet &LiveRegs_);300 // live regs for the current state301 const decltype(LiveRegs) &getLiveRegs() const { return LiveRegs; }302 const MachineInstr *getLastTrackedMI() const { return LastTrackedMI; }303 304 void clearMaxPressure() { MaxPressure.clear(); }305 306 GCNRegPressure getPressure() const { return CurPressure; }307 308 decltype(LiveRegs) moveLiveRegs() {309 return std::move(LiveRegs);310 }311};312 313GCNRPTracker::LiveRegSet314getLiveRegs(SlotIndex SI, const LiveIntervals &LIS,315 const MachineRegisterInfo &MRI,316 GCNRegPressure::RegKind RegKind = GCNRegPressure::TOTAL_KINDS);317 318////////////////////////////////////////////////////////////////////////////////319// GCNUpwardRPTracker320 321class GCNUpwardRPTracker : public GCNRPTracker {322public:323 GCNUpwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}324 325 using GCNRPTracker::reset;326 327 /// reset tracker at the specified slot index \p SI.328 void reset(const MachineRegisterInfo &MRI, SlotIndex SI) {329 GCNRPTracker::reset(MRI, llvm::getLiveRegs(SI, LIS, MRI));330 }331 332 /// reset tracker to the end of the \p MBB.333 void reset(const MachineBasicBlock &MBB) {334 SlotIndex MBBLastSlot = LIS.getSlotIndexes()->getMBBLastIdx(&MBB);335 reset(MBB.getParent()->getRegInfo(), MBBLastSlot);336 }337 338 /// reset tracker to the point just after \p MI (in program order).339 void reset(const MachineInstr &MI) {340 reset(MI.getMF()->getRegInfo(), LIS.getInstructionIndex(MI).getDeadSlot());341 }342 343 /// Move to the state of RP just before the \p MI . If \p UseInternalIterator344 /// is set, also update the internal iterators. Setting \p UseInternalIterator345 /// to false allows for an externally managed iterator / program order.346 void recede(const MachineInstr &MI);347 348 /// \p returns whether the tracker's state after receding MI corresponds349 /// to reported by LIS.350 bool isValid() const;351 352 const GCNRegPressure &getMaxPressure() const { return MaxPressure; }353 354 void resetMaxPressure() { MaxPressure = CurPressure; }355 356 GCNRegPressure getMaxPressureAndReset() {357 GCNRegPressure RP = MaxPressure;358 resetMaxPressure();359 return RP;360 }361};362 363////////////////////////////////////////////////////////////////////////////////364// GCNDownwardRPTracker365 366class GCNDownwardRPTracker : public GCNRPTracker {367 // Last position of reset or advanceBeforeNext368 MachineBasicBlock::const_iterator NextMI;369 370 MachineBasicBlock::const_iterator MBBEnd;371 372public:373 GCNDownwardRPTracker(const LiveIntervals &LIS_) : GCNRPTracker(LIS_) {}374 375 using GCNRPTracker::reset;376 377 MachineBasicBlock::const_iterator getNext() const { return NextMI; }378 379 /// \p return MaxPressure and clear it.380 GCNRegPressure moveMaxPressure() {381 auto Res = MaxPressure;382 MaxPressure.clear();383 return Res;384 }385 386 /// Reset tracker to the point before the \p MI387 /// filling \p LiveRegs upon this point using LIS.388 /// \p returns false if block is empty except debug values.389 bool reset(const MachineInstr &MI, const LiveRegSet *LiveRegs = nullptr);390 391 /// Move to the state right before the next MI or after the end of MBB.392 /// \p returns false if reached end of the block.393 /// If \p UseInternalIterator is true, then internal iterators are used and394 /// set to process in program order. If \p UseInternalIterator is false, then395 /// it is assumed that the tracker is using an externally managed iterator,396 /// and advance* calls will not update the state of the iterator. In such397 /// cases, the tracker will move to the state right before the provided \p MI398 /// and use LIS for RP calculations.399 bool advanceBeforeNext(MachineInstr *MI = nullptr,400 bool UseInternalIterator = true);401 402 /// Move to the state at the MI, advanceBeforeNext has to be called first.403 /// If \p UseInternalIterator is true, then internal iterators are used and404 /// set to process in program order. If \p UseInternalIterator is false, then405 /// it is assumed that the tracker is using an externally managed iterator,406 /// and advance* calls will not update the state of the iterator. In such407 /// cases, the tracker will move to the state at the provided \p MI .408 void advanceToNext(MachineInstr *MI = nullptr,409 bool UseInternalIterator = true);410 411 /// Move to the state at the next MI. \p returns false if reached end of412 /// block. If \p UseInternalIterator is true, then internal iterators are used413 /// and set to process in program order. If \p UseInternalIterator is false,414 /// then it is assumed that the tracker is using an externally managed415 /// iterator, and advance* calls will not update the state of the iterator. In416 /// such cases, the tracker will move to the state right before the provided417 /// \p MI and use LIS for RP calculations.418 bool advance(MachineInstr *MI = nullptr, bool UseInternalIterator = true);419 420 /// Advance instructions until before \p End.421 bool advance(MachineBasicBlock::const_iterator End);422 423 /// Reset to \p Begin and advance to \p End.424 bool advance(MachineBasicBlock::const_iterator Begin,425 MachineBasicBlock::const_iterator End,426 const LiveRegSet *LiveRegsCopy = nullptr);427 428 /// Mostly copy/paste from CodeGen/RegisterPressure.cpp429 /// Calculate the impact \p MI will have on CurPressure and \return the430 /// speculated pressure. In order to support RP Speculation, this does not431 /// rely on the implicit program ordering in the LiveIntervals.432 GCNRegPressure bumpDownwardPressure(const MachineInstr *MI,433 const SIRegisterInfo *TRI) const;434};435 436/// \returns the LaneMask of live lanes of \p Reg at position \p SI. Only the437/// active lanes of \p LaneMaskFilter will be set in the return value. This is438/// used, for example, to limit the live lanes to a specific subreg when439/// calculating use masks.440LaneBitmask getLiveLaneMask(unsigned Reg, SlotIndex SI,441 const LiveIntervals &LIS,442 const MachineRegisterInfo &MRI,443 LaneBitmask LaneMaskFilter = LaneBitmask::getAll());444 445LaneBitmask getLiveLaneMask(const LiveInterval &LI, SlotIndex SI,446 const MachineRegisterInfo &MRI,447 LaneBitmask LaneMaskFilter = LaneBitmask::getAll());448 449/// creates a map MachineInstr -> LiveRegSet450/// R - range of iterators on instructions451/// After - upon entry or exit of every instruction452/// Note: there is no entry in the map for instructions with empty live reg set453/// Complexity = O(NumVirtRegs * averageLiveRangeSegmentsPerReg * lg(R))454template <typename Range>455DenseMap<MachineInstr*, GCNRPTracker::LiveRegSet>456getLiveRegMap(Range &&R, bool After, LiveIntervals &LIS) {457 std::vector<SlotIndex> Indexes;458 Indexes.reserve(llvm::size(R));459 auto &SII = *LIS.getSlotIndexes();460 for (MachineInstr *I : R) {461 auto SI = SII.getInstructionIndex(*I);462 Indexes.push_back(After ? SI.getDeadSlot() : SI.getBaseIndex());463 }464 llvm::sort(Indexes);465 466 auto &MRI = (*R.begin())->getMF()->getRegInfo();467 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet> LiveRegMap;468 SmallVector<SlotIndex, 32> LiveIdxs, SRLiveIdxs;469 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {470 auto Reg = Register::index2VirtReg(I);471 if (!LIS.hasInterval(Reg))472 continue;473 auto &LI = LIS.getInterval(Reg);474 LiveIdxs.clear();475 if (!LI.findIndexesLiveAt(Indexes, std::back_inserter(LiveIdxs)))476 continue;477 if (!LI.hasSubRanges()) {478 for (auto SI : LiveIdxs)479 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] =480 MRI.getMaxLaneMaskForVReg(Reg);481 } else482 for (const auto &S : LI.subranges()) {483 // constrain search for subranges by indexes live at main range484 SRLiveIdxs.clear();485 S.findIndexesLiveAt(LiveIdxs, std::back_inserter(SRLiveIdxs));486 for (auto SI : SRLiveIdxs)487 LiveRegMap[SII.getInstructionFromIndex(SI)][Reg] |= S.LaneMask;488 }489 }490 return LiveRegMap;491}492 493inline GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI,494 const LiveIntervals &LIS) {495 return getLiveRegs(LIS.getInstructionIndex(MI).getDeadSlot(), LIS,496 MI.getMF()->getRegInfo());497}498 499inline GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI,500 const LiveIntervals &LIS) {501 return getLiveRegs(LIS.getInstructionIndex(MI).getBaseIndex(), LIS,502 MI.getMF()->getRegInfo());503}504 505template <typename Range>506GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI,507 Range &&LiveRegs) {508 GCNRegPressure Res;509 for (const auto &RM : LiveRegs)510 Res.inc(RM.first, LaneBitmask::getNone(), RM.second, MRI);511 return Res;512}513 514bool isEqual(const GCNRPTracker::LiveRegSet &S1,515 const GCNRPTracker::LiveRegSet &S2);516 517Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST = nullptr,518 unsigned DynamicVGPRBlockSize = 0);519 520Printable print(const GCNRPTracker::LiveRegSet &LiveRegs,521 const MachineRegisterInfo &MRI);522 523Printable reportMismatch(const GCNRPTracker::LiveRegSet &LISLR,524 const GCNRPTracker::LiveRegSet &TrackedL,525 const TargetRegisterInfo *TRI, StringRef Pfx = " ");526 527struct GCNRegPressurePrinter : public MachineFunctionPass {528 static char ID;529 530public:531 GCNRegPressurePrinter() : MachineFunctionPass(ID) {}532 533 bool runOnMachineFunction(MachineFunction &MF) override;534 535 void getAnalysisUsage(AnalysisUsage &AU) const override {536 AU.addRequired<LiveIntervalsWrapperPass>();537 AU.setPreservesAll();538 MachineFunctionPass::getAnalysisUsage(AU);539 }540};541 542LLVM_ABI void dumpMaxRegPressure(MachineFunction &MF,543 GCNRegPressure::RegKind Kind,544 LiveIntervals &LIS,545 const MachineLoopInfo *MLI);546 547} // end namespace llvm548 549#endif // LLVM_LIB_TARGET_AMDGPU_GCNREGPRESSURE_H550