671 lines · cpp
1//===-- AMDGPURewriteAGPRCopyMFMA.cpp -------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file \brief Try to replace MFMA instructions using VGPRs with MFMA10/// instructions using AGPRs. We expect MFMAs to be selected using VGPRs, and11/// only use AGPRs if it helps avoid spilling. In this case, the MFMA will have12/// copies between AGPRs and VGPRs and the AGPR variant of an MFMA pseudo. This13/// pass will attempt to delete the cross register bank copy and replace the14/// MFMA opcode.15///16/// TODO:17/// - Handle rewrites of phis. This must be more careful than normal about the18/// reassignment. We do not want to introduce an AGPR-to-AGPR copy inside of a19/// loop, so it depends on the exact assignment of the copy.20///21/// - Update LiveIntervals incrementally instead of recomputing from scratch22///23//===----------------------------------------------------------------------===//24 25#include "AMDGPU.h"26#include "GCNSubtarget.h"27#include "SIMachineFunctionInfo.h"28#include "SIRegisterInfo.h"29#include "llvm/ADT/Statistic.h"30#include "llvm/CodeGen/LiveIntervals.h"31#include "llvm/CodeGen/LiveRegMatrix.h"32#include "llvm/CodeGen/LiveStacks.h"33#include "llvm/CodeGen/MachineFrameInfo.h"34#include "llvm/CodeGen/MachineFunctionPass.h"35#include "llvm/CodeGen/SlotIndexes.h"36#include "llvm/CodeGen/VirtRegMap.h"37#include "llvm/InitializePasses.h"38 39using namespace llvm;40 41#define DEBUG_TYPE "amdgpu-rewrite-agpr-copy-mfma"42 43namespace {44 45STATISTIC(NumMFMAsRewrittenToAGPR,46 "Number of MFMA instructions rewritten to use AGPR form");47 48/// Map from spill slot frame index to list of instructions which reference it.49using SpillReferenceMap = DenseMap<int, SmallVector<MachineInstr *, 4>>;50 51class AMDGPURewriteAGPRCopyMFMAImpl {52 MachineFunction &MF;53 const GCNSubtarget &ST;54 const SIInstrInfo &TII;55 const SIRegisterInfo &TRI;56 MachineRegisterInfo &MRI;57 VirtRegMap &VRM;58 LiveRegMatrix &LRM;59 LiveIntervals &LIS;60 LiveStacks &LSS;61 const RegisterClassInfo &RegClassInfo;62 63 bool attemptReassignmentsToAGPR(SmallSetVector<Register, 4> &InterferingRegs,64 MCPhysReg PrefPhysReg) const;65 66public:67 AMDGPURewriteAGPRCopyMFMAImpl(MachineFunction &MF, VirtRegMap &VRM,68 LiveRegMatrix &LRM, LiveIntervals &LIS,69 LiveStacks &LSS,70 const RegisterClassInfo &RegClassInfo)71 : MF(MF), ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),72 TRI(*ST.getRegisterInfo()), MRI(MF.getRegInfo()), VRM(VRM), LRM(LRM),73 LIS(LIS), LSS(LSS), RegClassInfo(RegClassInfo) {}74 75 bool isRewriteCandidate(const MachineInstr &MI) const {76 return TII.isMAI(MI) && AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode()) != -1;77 }78 79 /// Find AV_* registers assigned to AGPRs (or virtual registers which were80 /// already required to be AGPR).81 ///82 /// \return the assigned physical register that \p VReg is assigned to if it83 /// is an AGPR, otherwise MCRegister().84 MCRegister getAssignedAGPR(Register VReg) const {85 MCRegister PhysReg = VRM.getPhys(VReg);86 if (!PhysReg)87 return MCRegister();88 89 // If this is an AV register, we have to check if the actual assignment is90 // to an AGPR91 const TargetRegisterClass *AssignedRC = TRI.getPhysRegBaseClass(PhysReg);92 return TRI.isAGPRClass(AssignedRC) ? PhysReg : MCRegister();93 }94 95 bool tryReassigningMFMAChain(MachineInstr &MFMA, Register MFMAHintReg,96 MCPhysReg PhysRegHint) const;97 98 /// Compute the register class constraints based on the uses of \p Reg,99 /// excluding MFMA uses from which can be rewritten to change the register100 /// class constraint. This should be nearly identical to101 /// MachineRegisterInfo::recomputeRegClass.102 103 /// \p RewriteCandidates will collect the set of MFMA instructions that need104 /// to have the opcode mutated to perform the replacement.105 ///106 /// \p RewriteRegs will accumulate the set of register used by those MFMAs107 /// that need to have the register classes adjusted.108 bool recomputeRegClassExceptRewritable(109 Register Reg, SmallVectorImpl<MachineInstr *> &RewriteCandidates,110 SmallSetVector<Register, 4> &RewriteRegs) const;111 112 bool tryFoldCopiesToAGPR(Register VReg, MCRegister AssignedAGPR) const;113 bool tryFoldCopiesFromAGPR(Register VReg, MCRegister AssignedAGPR) const;114 115 /// Replace spill instruction \p SpillMI which loads/stores from/to \p SpillFI116 /// with a COPY to the replacement register value \p VReg.117 void replaceSpillWithCopyToVReg(MachineInstr &SpillMI, int SpillFI,118 Register VReg) const;119 120 /// Create a map from frame index to use instructions for spills. If a use of121 /// the frame index does not consist only of spill instructions, it will not122 /// be included in the map.123 void collectSpillIndexUses(ArrayRef<LiveInterval *> StackIntervals,124 SpillReferenceMap &Map) const;125 126 /// Attempt to unspill VGPRs by finding a free register and replacing the127 /// spill instructions with copies.128 void eliminateSpillsOfReassignedVGPRs() const;129 130 bool run(MachineFunction &MF) const;131};132 133bool AMDGPURewriteAGPRCopyMFMAImpl::recomputeRegClassExceptRewritable(134 Register StartReg, SmallVectorImpl<MachineInstr *> &RewriteCandidates,135 SmallSetVector<Register, 4> &RewriteRegs) const {136 SmallVector<Register, 8> Worklist = {StartReg};137 138 // Recursively visit all transitive MFMA users139 while (!Worklist.empty()) {140 Register Reg = Worklist.pop_back_val();141 const TargetRegisterClass *OldRC = MRI.getRegClass(Reg);142 143 // Inflate to the equivalent AV_* class.144 const TargetRegisterClass *NewRC = TRI.getLargestLegalSuperClass(OldRC, MF);145 if (OldRC == NewRC)146 return false;147 148 // Accumulate constraints from all uses.149 for (MachineOperand &MO : MRI.reg_nodbg_operands(Reg)) {150 // Apply the effect of the given operand to NewRC.151 MachineInstr *MI = MO.getParent();152 153 // We can swap the classes of dst + src2 as a pair to AGPR, so ignore the154 // effects of rewrite candidates. It just so happens that we can use155 // either AGPR or VGPR in src0/src1, so don't bother checking the156 // constraint effects of the individual operands.157 if (isRewriteCandidate(*MI)) {158 const MachineOperand *VDst =159 TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);160 const MachineOperand *Src2 =161 TII.getNamedOperand(*MI, AMDGPU::OpName::src2);162 for (const MachineOperand *Op : {VDst, Src2}) {163 if (!Op->isReg())164 continue;165 166 Register OtherReg = Op->getReg();167 if (OtherReg.isPhysical())168 return false;169 170 if (OtherReg != Reg && RewriteRegs.insert(OtherReg))171 Worklist.push_back(OtherReg);172 }173 174 if (!is_contained(RewriteCandidates, MI)) {175 LLVM_DEBUG({176 Register VDstPhysReg = VRM.getPhys(VDst->getReg());177 dbgs() << "Attempting to replace VGPR MFMA with AGPR version:"178 << " Dst=[" << printReg(VDst->getReg()) << " => "179 << printReg(VDstPhysReg, &TRI);180 181 if (Src2->isReg()) {182 Register Src2PhysReg = VRM.getPhys(Src2->getReg());183 dbgs() << "], Src2=[" << printReg(Src2->getReg(), &TRI) << " => "184 << printReg(Src2PhysReg, &TRI);185 }186 187 dbgs() << "]: " << MI;188 });189 190 RewriteCandidates.push_back(MI);191 }192 193 continue;194 }195 196 unsigned OpNo = &MO - &MI->getOperand(0);197 NewRC = MI->getRegClassConstraintEffect(OpNo, NewRC, &TII, &TRI);198 if (!NewRC || NewRC == OldRC) {199 LLVM_DEBUG(dbgs() << "User of " << printReg(Reg, &TRI)200 << " cannot be reassigned to "201 << TRI.getRegClassName(NewRC) << ": " << *MI);202 return false;203 }204 }205 }206 207 return true;208}209 210bool AMDGPURewriteAGPRCopyMFMAImpl::tryReassigningMFMAChain(211 MachineInstr &MFMA, Register MFMAHintReg, MCPhysReg PhysRegHint) const {212 // src2 and dst have the same physical class constraint; try to preserve213 // the original src2 subclass if one were to exist.214 SmallVector<MachineInstr *, 4> RewriteCandidates = {&MFMA};215 SmallSetVector<Register, 4> RewriteRegs;216 217 // Make sure we reassign the MFMA we found the copy from first. We want218 // to ensure dst ends up in the physreg we were originally copying to.219 RewriteRegs.insert(MFMAHintReg);220 221 // We've found av = COPY (MFMA) (or MFMA (v = COPY av)) and need to verify222 // that we can trivially rewrite src2 to use the new AGPR. If we can't223 // trivially replace it, we're going to induce as many copies as we would have224 // emitted in the first place, as well as need to assign another register, and225 // need to figure out where to put them. The live range splitting is smarter226 // than anything we're doing here, so trust it did something reasonable.227 //228 // Note recomputeRegClassExceptRewritable will consider the constraints of229 // this MFMA's src2 as well as the src2/dst of any transitive MFMA users.230 if (!recomputeRegClassExceptRewritable(MFMAHintReg, RewriteCandidates,231 RewriteRegs)) {232 LLVM_DEBUG(dbgs() << "Could not recompute the regclass of dst reg "233 << printReg(MFMAHintReg, &TRI) << '\n');234 return false;235 }236 237 // If src2 and dst are different registers, we need to also reassign the238 // input to an available AGPR if it is compatible with all other uses.239 //240 // If we can't reassign it, we'd need to introduce a different copy241 // which is likely worse than the copy we'd be saving.242 //243 // It's likely that the MFMA is used in sequence with other MFMAs; if we244 // cannot migrate the full use/def chain of MFMAs, we would need to245 // introduce intermediate copies somewhere. So we only make the246 // transform if all the interfering MFMAs can also be migrated. Collect247 // the set of rewritable MFMAs and check if we can assign an AGPR at248 // that point.249 //250 // If any of the MFMAs aren't reassignable, we give up and rollback to251 // the original register assignments.252 253 using RecoloringStack =254 SmallVector<std::pair<const LiveInterval *, MCRegister>, 8>;255 RecoloringStack TentativeReassignments;256 257 for (Register RewriteReg : RewriteRegs) {258 LiveInterval &LI = LIS.getInterval(RewriteReg);259 TentativeReassignments.push_back({&LI, VRM.getPhys(RewriteReg)});260 LRM.unassign(LI);261 }262 263 if (!attemptReassignmentsToAGPR(RewriteRegs, PhysRegHint)) {264 // Roll back the register assignments to the original state.265 for (auto [LI, OldAssign] : TentativeReassignments) {266 if (VRM.hasPhys(LI->reg()))267 LRM.unassign(*LI);268 LRM.assign(*LI, OldAssign);269 }270 271 return false;272 }273 274 // Fixup the register classes of the virtual registers now that we've275 // committed to the reassignments.276 for (Register InterferingReg : RewriteRegs) {277 const TargetRegisterClass *EquivalentAGPRRegClass =278 TRI.getEquivalentAGPRClass(MRI.getRegClass(InterferingReg));279 MRI.setRegClass(InterferingReg, EquivalentAGPRRegClass);280 }281 282 for (MachineInstr *RewriteCandidate : RewriteCandidates) {283 int NewMFMAOp =284 AMDGPU::getMFMASrcCVDstAGPROp(RewriteCandidate->getOpcode());285 RewriteCandidate->setDesc(TII.get(NewMFMAOp));286 ++NumMFMAsRewrittenToAGPR;287 }288 289 return true;290}291 292/// Attempt to reassign the registers in \p InterferingRegs to be AGPRs, with a293/// preference to use \p PhysReg first. Returns false if the reassignments294/// cannot be trivially performed.295bool AMDGPURewriteAGPRCopyMFMAImpl::attemptReassignmentsToAGPR(296 SmallSetVector<Register, 4> &InterferingRegs, MCPhysReg PrefPhysReg) const {297 // FIXME: The ordering may matter here, but we're just taking uselistorder298 // with the special case of ensuring to process the starting instruction299 // first. We probably should extract the priority advisor out of greedy and300 // use that ordering.301 for (Register InterferingReg : InterferingRegs) {302 LiveInterval &ReassignLI = LIS.getInterval(InterferingReg);303 const TargetRegisterClass *EquivalentAGPRRegClass =304 TRI.getEquivalentAGPRClass(MRI.getRegClass(InterferingReg));305 306 MCPhysReg Assignable = AMDGPU::NoRegister;307 if (EquivalentAGPRRegClass->contains(PrefPhysReg) &&308 LRM.checkInterference(ReassignLI, PrefPhysReg) ==309 LiveRegMatrix::IK_Free) {310 // First try to assign to the AGPR we were already copying to. This311 // should be the first assignment we attempt. We have to guard312 // against the use being a subregister (which doesn't have an exact313 // class match).314 315 // TODO: If this does happen to be a subregister use, we should316 // still try to assign to a subregister of the original copy result.317 Assignable = PrefPhysReg;318 } else {319 ArrayRef<MCPhysReg> AllocOrder =320 RegClassInfo.getOrder(EquivalentAGPRRegClass);321 for (MCPhysReg Reg : AllocOrder) {322 if (LRM.checkInterference(ReassignLI, Reg) == LiveRegMatrix::IK_Free) {323 Assignable = Reg;324 break;325 }326 }327 }328 329 if (!Assignable) {330 LLVM_DEBUG(dbgs() << "Unable to reassign VGPR "331 << printReg(InterferingReg, &TRI)332 << " to a free AGPR\n");333 return false;334 }335 336 LLVM_DEBUG(dbgs() << "Reassigning VGPR " << printReg(InterferingReg, &TRI)337 << " to " << printReg(Assignable, &TRI) << '\n');338 LRM.assign(ReassignLI, Assignable);339 }340 341 return true;342}343 344/// Identify copies that look like:345/// %vdst:vgpr = V_MFMA_.. %src0:av, %src1:av, %src2:vgpr346/// %agpr = COPY %vgpr347///348/// Then try to replace the transitive uses of %src2 and %vdst with the AGPR349/// versions of the MFMA. This should cover the common case.350bool AMDGPURewriteAGPRCopyMFMAImpl::tryFoldCopiesToAGPR(351 Register VReg, MCRegister AssignedAGPR) const {352 bool MadeChange = false;353 for (MachineInstr &UseMI : MRI.def_instructions(VReg)) {354 if (!UseMI.isCopy())355 continue;356 357 Register CopySrcReg = UseMI.getOperand(1).getReg();358 if (!CopySrcReg.isVirtual())359 continue;360 361 // TODO: Handle loop phis copied to AGPR. e.g.362 //363 // loop:364 // %phi:vgpr = COPY %mfma:vgpr365 // %mfma:vgpr = V_MFMA_xxx_vgprcd_e64 %a, %b, %phi366 // s_cbranch_vccnz loop367 //368 // endloop:369 // %agpr = mfma370 //371 // We need to be sure that %phi is assigned to the same physical register as372 // %mfma, or else we will just be moving copies into the loop.373 374 for (MachineInstr &CopySrcDefMI : MRI.def_instructions(CopySrcReg)) {375 if (isRewriteCandidate(CopySrcDefMI) &&376 tryReassigningMFMAChain(377 CopySrcDefMI, CopySrcDefMI.getOperand(0).getReg(), AssignedAGPR))378 MadeChange = true;379 }380 }381 382 return MadeChange;383}384 385/// Identify copies that look like:386/// %src:vgpr = COPY %src:agpr387/// %vdst:vgpr = V_MFMA_... %src0:av, %src1:av, %src:vgpr388///389/// Then try to replace the transitive uses of %src2 and %vdst with the AGPR390/// versions of the MFMA. This should cover rarer cases, and will generally be391/// redundant with tryFoldCopiesToAGPR.392bool AMDGPURewriteAGPRCopyMFMAImpl::tryFoldCopiesFromAGPR(393 Register VReg, MCRegister AssignedAGPR) const {394 bool MadeChange = false;395 for (MachineInstr &UseMI : MRI.use_instructions(VReg)) {396 if (!UseMI.isCopy())397 continue;398 399 Register CopyDstReg = UseMI.getOperand(0).getReg();400 if (!CopyDstReg.isVirtual())401 continue;402 for (MachineOperand &CopyUseMO : MRI.reg_nodbg_operands(CopyDstReg)) {403 if (!CopyUseMO.readsReg())404 continue;405 406 MachineInstr &CopyUseMI = *CopyUseMO.getParent();407 if (isRewriteCandidate(CopyUseMI)) {408 if (tryReassigningMFMAChain(CopyUseMI, CopyDstReg,409 VRM.getPhys(CopyDstReg)))410 MadeChange = true;411 }412 }413 }414 415 return MadeChange;416}417 418void AMDGPURewriteAGPRCopyMFMAImpl::replaceSpillWithCopyToVReg(419 MachineInstr &SpillMI, int SpillFI, Register VReg) const {420 const DebugLoc &DL = SpillMI.getDebugLoc();421 MachineBasicBlock &MBB = *SpillMI.getParent();422 MachineInstr *NewCopy;423 if (SpillMI.mayStore()) {424 NewCopy = BuildMI(MBB, SpillMI, DL, TII.get(TargetOpcode::COPY), VReg)425 .add(SpillMI.getOperand(0));426 } else {427 NewCopy = BuildMI(MBB, SpillMI, DL, TII.get(TargetOpcode::COPY))428 .add(SpillMI.getOperand(0))429 .addReg(VReg);430 }431 432 LIS.ReplaceMachineInstrInMaps(SpillMI, *NewCopy);433 SpillMI.eraseFromParent();434}435 436void AMDGPURewriteAGPRCopyMFMAImpl::collectSpillIndexUses(437 ArrayRef<LiveInterval *> StackIntervals, SpillReferenceMap &Map) const {438 439 SmallSet<int, 4> NeededFrameIndexes;440 for (const LiveInterval *LI : StackIntervals)441 NeededFrameIndexes.insert(LI->reg().stackSlotIndex());442 443 for (MachineBasicBlock &MBB : MF) {444 for (MachineInstr &MI : MBB) {445 for (MachineOperand &MO : MI.operands()) {446 if (!MO.isFI() || !NeededFrameIndexes.count(MO.getIndex()))447 continue;448 449 if (TII.isVGPRSpill(MI)) {450 SmallVector<MachineInstr *, 4> &References = Map[MO.getIndex()];451 References.push_back(&MI);452 break;453 }454 455 // Verify this was really a spill instruction, if it's not just ignore456 // all uses.457 458 // TODO: This should probably be verifier enforced.459 NeededFrameIndexes.erase(MO.getIndex());460 Map.erase(MO.getIndex());461 }462 }463 }464}465 466void AMDGPURewriteAGPRCopyMFMAImpl::eliminateSpillsOfReassignedVGPRs() const {467 unsigned NumSlots = LSS.getNumIntervals();468 if (NumSlots == 0)469 return;470 471 MachineFrameInfo &MFI = MF.getFrameInfo();472 473 SmallVector<LiveInterval *, 32> StackIntervals;474 StackIntervals.reserve(NumSlots);475 476 for (auto &[Slot, LI] : LSS) {477 if (!MFI.isSpillSlotObjectIndex(Slot) || MFI.isDeadObjectIndex(Slot))478 continue;479 480 const TargetRegisterClass *RC = LSS.getIntervalRegClass(Slot);481 if (TRI.hasVGPRs(RC))482 StackIntervals.push_back(&LI);483 }484 485 sort(StackIntervals, [](const LiveInterval *A, const LiveInterval *B) {486 // The ordering has to be strictly weak.487 /// Sort heaviest intervals first to prioritize their unspilling488 if (A->weight() != B->weight())489 return A->weight() > B->weight();490 491 if (A->getSize() != B->getSize())492 return A->getSize() > B->getSize();493 494 // Tie breaker by number to avoid need for stable sort495 return A->reg().stackSlotIndex() < B->reg().stackSlotIndex();496 });497 498 // FIXME: The APIs for dealing with the LiveInterval of a frame index are499 // cumbersome. LiveStacks owns its LiveIntervals which refer to stack500 // slots. We cannot use the usual LiveRegMatrix::assign and unassign on these,501 // and must create a substitute virtual register to do so. This makes502 // incremental updating here difficult; we need to actually perform the IR503 // mutation to get the new vreg references in place to compute the register504 // LiveInterval to perform an assignment to track the new interference505 // correctly, and we can't simply migrate the LiveInterval we already have.506 //507 // To avoid walking through the entire function for each index, pre-collect508 // all the instructions slot referencess.509 510 DenseMap<int, SmallVector<MachineInstr *, 4>> SpillSlotReferences;511 collectSpillIndexUses(StackIntervals, SpillSlotReferences);512 513 for (LiveInterval *LI : StackIntervals) {514 int Slot = LI->reg().stackSlotIndex();515 auto SpillReferences = SpillSlotReferences.find(Slot);516 if (SpillReferences == SpillSlotReferences.end())517 continue;518 519 const TargetRegisterClass *RC = LSS.getIntervalRegClass(Slot);520 521 LLVM_DEBUG(dbgs() << "Trying to eliminate " << printReg(Slot, &TRI)522 << " by reassigning\n");523 524 ArrayRef<MCPhysReg> AllocOrder = RegClassInfo.getOrder(RC);525 526 for (MCPhysReg PhysReg : AllocOrder) {527 if (LRM.checkInterference(*LI, PhysReg) != LiveRegMatrix::IK_Free)528 continue;529 530 LLVM_DEBUG(dbgs() << "Reassigning " << *LI << " to "531 << printReg(PhysReg, &TRI) << '\n');532 533 const TargetRegisterClass *RC = LSS.getIntervalRegClass(Slot);534 Register NewVReg = MRI.createVirtualRegister(RC);535 536 for (MachineInstr *SpillMI : SpillReferences->second)537 replaceSpillWithCopyToVReg(*SpillMI, Slot, NewVReg);538 539 // TODO: We should be able to transfer the information from the stack540 // slot's LiveInterval without recomputing from scratch with the541 // replacement vreg uses.542 LiveInterval &NewLI = LIS.createAndComputeVirtRegInterval(NewVReg);543 VRM.grow();544 LRM.assign(NewLI, PhysReg);545 MFI.RemoveStackObject(Slot);546 break;547 }548 }549}550 551bool AMDGPURewriteAGPRCopyMFMAImpl::run(MachineFunction &MF) const {552 // This only applies on subtargets that have a configurable AGPR vs. VGPR553 // allocation.554 if (!ST.hasGFX90AInsts())555 return false;556 557 // Early exit if no AGPRs were assigned.558 if (!LRM.isPhysRegUsed(AMDGPU::AGPR0)) {559 LLVM_DEBUG(dbgs() << "skipping function that did not allocate AGPRs\n");560 return false;561 }562 563 bool MadeChange = false;564 565 for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {566 Register VReg = Register::index2VirtReg(I);567 MCRegister AssignedAGPR = getAssignedAGPR(VReg);568 if (!AssignedAGPR)569 continue;570 571 if (tryFoldCopiesToAGPR(VReg, AssignedAGPR))572 MadeChange = true;573 if (tryFoldCopiesFromAGPR(VReg, AssignedAGPR))574 MadeChange = true;575 }576 577 // If we've successfully rewritten some MFMAs, we've alleviated some VGPR578 // pressure. See if we can eliminate some spills now that those registers are579 // more available.580 if (MadeChange)581 eliminateSpillsOfReassignedVGPRs();582 583 return MadeChange;584}585 586class AMDGPURewriteAGPRCopyMFMALegacy : public MachineFunctionPass {587public:588 static char ID;589 RegisterClassInfo RegClassInfo;590 591 AMDGPURewriteAGPRCopyMFMALegacy() : MachineFunctionPass(ID) {592 initializeAMDGPURewriteAGPRCopyMFMALegacyPass(593 *PassRegistry::getPassRegistry());594 }595 596 bool runOnMachineFunction(MachineFunction &MF) override;597 598 StringRef getPassName() const override {599 return "AMDGPU Rewrite AGPR-Copy-MFMA";600 }601 602 void getAnalysisUsage(AnalysisUsage &AU) const override {603 AU.addRequired<LiveIntervalsWrapperPass>();604 AU.addRequired<VirtRegMapWrapperLegacy>();605 AU.addRequired<LiveRegMatrixWrapperLegacy>();606 AU.addRequired<LiveStacksWrapperLegacy>();607 608 AU.addPreserved<LiveIntervalsWrapperPass>();609 AU.addPreserved<VirtRegMapWrapperLegacy>();610 AU.addPreserved<LiveRegMatrixWrapperLegacy>();611 AU.addPreserved<LiveStacksWrapperLegacy>();612 613 AU.setPreservesAll();614 MachineFunctionPass::getAnalysisUsage(AU);615 }616};617 618} // End anonymous namespace.619 620INITIALIZE_PASS_BEGIN(AMDGPURewriteAGPRCopyMFMALegacy, DEBUG_TYPE,621 "AMDGPU Rewrite AGPR-Copy-MFMA", false, false)622INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)623INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)624INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy)625INITIALIZE_PASS_DEPENDENCY(LiveStacksWrapperLegacy)626INITIALIZE_PASS_END(AMDGPURewriteAGPRCopyMFMALegacy, DEBUG_TYPE,627 "AMDGPU Rewrite AGPR-Copy-MFMA", false, false)628 629char AMDGPURewriteAGPRCopyMFMALegacy::ID = 0;630 631char &llvm::AMDGPURewriteAGPRCopyMFMALegacyID =632 AMDGPURewriteAGPRCopyMFMALegacy::ID;633 634bool AMDGPURewriteAGPRCopyMFMALegacy::runOnMachineFunction(635 MachineFunction &MF) {636 if (skipFunction(MF.getFunction()))637 return false;638 639 RegClassInfo.runOnMachineFunction(MF);640 641 auto &VRM = getAnalysis<VirtRegMapWrapperLegacy>().getVRM();642 auto &LRM = getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();643 auto &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();644 auto &LSS = getAnalysis<LiveStacksWrapperLegacy>().getLS();645 AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS, LSS, RegClassInfo);646 return Impl.run(MF);647}648 649PreservedAnalyses650AMDGPURewriteAGPRCopyMFMAPass::run(MachineFunction &MF,651 MachineFunctionAnalysisManager &MFAM) {652 VirtRegMap &VRM = MFAM.getResult<VirtRegMapAnalysis>(MF);653 LiveRegMatrix &LRM = MFAM.getResult<LiveRegMatrixAnalysis>(MF);654 LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);655 LiveStacks &LSS = MFAM.getResult<LiveStacksAnalysis>(MF);656 RegisterClassInfo RegClassInfo;657 RegClassInfo.runOnMachineFunction(MF);658 659 AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS, LSS, RegClassInfo);660 if (!Impl.run(MF))661 return PreservedAnalyses::all();662 auto PA = getMachineFunctionPassPreservedAnalyses();663 PA.preserveSet<CFGAnalyses>()664 .preserve<LiveStacksAnalysis>()665 .preserve<VirtRegMapAnalysis>()666 .preserve<SlotIndexesAnalysis>()667 .preserve<LiveIntervalsAnalysis>()668 .preserve<LiveRegMatrixAnalysis>();669 return PA;670}671