275 lines · cpp
1//===- SIPreAllocateWWMRegs.cpp - WWM Register Pre-allocation -------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// Pass to pre-allocated WWM registers11//12//===----------------------------------------------------------------------===//13 14#include "SIPreAllocateWWMRegs.h"15#include "AMDGPU.h"16#include "GCNSubtarget.h"17#include "MCTargetDesc/AMDGPUMCTargetDesc.h"18#include "SIMachineFunctionInfo.h"19#include "llvm/ADT/PostOrderIterator.h"20#include "llvm/CodeGen/LiveIntervals.h"21#include "llvm/CodeGen/LiveRegMatrix.h"22#include "llvm/CodeGen/MachineFrameInfo.h"23#include "llvm/CodeGen/MachineFunctionPass.h"24#include "llvm/CodeGen/RegisterClassInfo.h"25#include "llvm/CodeGen/VirtRegMap.h"26#include "llvm/InitializePasses.h"27 28using namespace llvm;29 30#define DEBUG_TYPE "si-pre-allocate-wwm-regs"31 32static cl::opt<bool>33 EnablePreallocateSGPRSpillVGPRs("amdgpu-prealloc-sgpr-spill-vgprs",34 cl::init(false), cl::Hidden);35 36namespace {37 38class SIPreAllocateWWMRegs {39private:40 const SIInstrInfo *TII;41 const SIRegisterInfo *TRI;42 MachineRegisterInfo *MRI;43 LiveIntervals *LIS;44 LiveRegMatrix *Matrix;45 VirtRegMap *VRM;46 RegisterClassInfo RegClassInfo;47 48 std::vector<unsigned> RegsToRewrite;49#ifndef NDEBUG50 void printWWMInfo(const MachineInstr &MI);51#endif52 bool processDef(MachineOperand &MO);53 void rewriteRegs(MachineFunction &MF);54 55public:56 SIPreAllocateWWMRegs(LiveIntervals *LIS, LiveRegMatrix *Matrix,57 VirtRegMap *VRM)58 : LIS(LIS), Matrix(Matrix), VRM(VRM) {}59 bool run(MachineFunction &MF);60};61 62class SIPreAllocateWWMRegsLegacy : public MachineFunctionPass {63public:64 static char ID;65 66 SIPreAllocateWWMRegsLegacy() : MachineFunctionPass(ID) {}67 68 bool runOnMachineFunction(MachineFunction &MF) override;69 70 void getAnalysisUsage(AnalysisUsage &AU) const override {71 AU.addRequired<LiveIntervalsWrapperPass>();72 AU.addRequired<VirtRegMapWrapperLegacy>();73 AU.addRequired<LiveRegMatrixWrapperLegacy>();74 AU.setPreservesAll();75 MachineFunctionPass::getAnalysisUsage(AU);76 }77};78 79} // End anonymous namespace.80 81INITIALIZE_PASS_BEGIN(SIPreAllocateWWMRegsLegacy, DEBUG_TYPE,82 "SI Pre-allocate WWM Registers", false, false)83INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)84INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)85INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy)86INITIALIZE_PASS_END(SIPreAllocateWWMRegsLegacy, DEBUG_TYPE,87 "SI Pre-allocate WWM Registers", false, false)88 89char SIPreAllocateWWMRegsLegacy::ID = 0;90 91char &llvm::SIPreAllocateWWMRegsLegacyID = SIPreAllocateWWMRegsLegacy::ID;92 93FunctionPass *llvm::createSIPreAllocateWWMRegsLegacyPass() {94 return new SIPreAllocateWWMRegsLegacy();95}96 97bool SIPreAllocateWWMRegs::processDef(MachineOperand &MO) {98 Register Reg = MO.getReg();99 if (Reg.isPhysical())100 return false;101 102 if (!TRI->isVGPR(*MRI, Reg))103 return false;104 105 if (VRM->hasPhys(Reg))106 return false;107 108 LiveInterval &LI = LIS->getInterval(Reg);109 110 for (MCRegister PhysReg : RegClassInfo.getOrder(MRI->getRegClass(Reg))) {111 if (!MRI->isPhysRegUsed(PhysReg, /*SkipRegMaskTest=*/true) &&112 Matrix->checkInterference(LI, PhysReg) == LiveRegMatrix::IK_Free) {113 Matrix->assign(LI, PhysReg);114 assert(PhysReg != 0);115 RegsToRewrite.push_back(Reg);116 return true;117 }118 }119 120 llvm_unreachable("physreg not found for WWM expression");121}122 123void SIPreAllocateWWMRegs::rewriteRegs(MachineFunction &MF) {124 for (MachineBasicBlock &MBB : MF) {125 for (MachineInstr &MI : MBB) {126 for (MachineOperand &MO : MI.operands()) {127 if (!MO.isReg())128 continue;129 130 const Register VirtReg = MO.getReg();131 if (VirtReg.isPhysical())132 continue;133 134 if (!VirtReg.isValid())135 continue;136 137 if (!VRM->hasPhys(VirtReg))138 continue;139 140 Register PhysReg = VRM->getPhys(VirtReg);141 const unsigned SubReg = MO.getSubReg();142 if (SubReg != 0) {143 PhysReg = TRI->getSubReg(PhysReg, SubReg);144 MO.setSubReg(0);145 }146 147 MO.setReg(PhysReg);148 MO.setIsRenamable(false);149 }150 }151 }152 153 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();154 155 for (unsigned Reg : RegsToRewrite) {156 LIS->removeInterval(Reg);157 158 const Register PhysReg = VRM->getPhys(Reg);159 assert(PhysReg != 0);160 161 MFI->reserveWWMRegister(PhysReg);162 }163 164 RegsToRewrite.clear();165 166 // Update the set of reserved registers to include WWM ones.167 MRI->freezeReservedRegs();168}169 170#ifndef NDEBUG171LLVM_DUMP_METHOD void172SIPreAllocateWWMRegs::printWWMInfo(const MachineInstr &MI) {173 174 unsigned Opc = MI.getOpcode();175 176 if (Opc == AMDGPU::ENTER_STRICT_WWM || Opc == AMDGPU::ENTER_STRICT_WQM) {177 dbgs() << "Entering ";178 } else {179 assert(Opc == AMDGPU::EXIT_STRICT_WWM || Opc == AMDGPU::EXIT_STRICT_WQM);180 dbgs() << "Exiting ";181 }182 183 if (Opc == AMDGPU::ENTER_STRICT_WWM || Opc == AMDGPU::EXIT_STRICT_WWM) {184 dbgs() << "Strict WWM ";185 } else {186 assert(Opc == AMDGPU::ENTER_STRICT_WQM || Opc == AMDGPU::EXIT_STRICT_WQM);187 dbgs() << "Strict WQM ";188 }189 190 dbgs() << "region: " << MI;191}192 193#endif194 195bool SIPreAllocateWWMRegsLegacy::runOnMachineFunction(MachineFunction &MF) {196 auto *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();197 auto *Matrix = &getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();198 auto *VRM = &getAnalysis<VirtRegMapWrapperLegacy>().getVRM();199 return SIPreAllocateWWMRegs(LIS, Matrix, VRM).run(MF);200}201 202bool SIPreAllocateWWMRegs::run(MachineFunction &MF) {203 LLVM_DEBUG(dbgs() << "SIPreAllocateWWMRegs: function " << MF.getName() << "\n");204 205 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();206 207 TII = ST.getInstrInfo();208 TRI = &TII->getRegisterInfo();209 MRI = &MF.getRegInfo();210 211 RegClassInfo.runOnMachineFunction(MF);212 213 bool PreallocateSGPRSpillVGPRs =214 EnablePreallocateSGPRSpillVGPRs ||215 MF.getFunction().hasFnAttribute("amdgpu-prealloc-sgpr-spill-vgprs");216 217 bool RegsAssigned = false;218 219 // We use a reverse post-order traversal of the control-flow graph to220 // guarantee that we visit definitions in dominance order. Since WWM221 // expressions are guaranteed to never involve phi nodes, and we can only222 // escape WWM through the special WWM instruction, this means that this is a223 // perfect elimination order, so we can never do any better.224 ReversePostOrderTraversal<MachineFunction*> RPOT(&MF);225 226 for (MachineBasicBlock *MBB : RPOT) {227 bool InWWM = false;228 for (MachineInstr &MI : *MBB) {229 if (MI.getOpcode() == AMDGPU::SI_SPILL_S32_TO_VGPR) {230 if (PreallocateSGPRSpillVGPRs)231 RegsAssigned |= processDef(MI.getOperand(0));232 continue;233 }234 235 if (MI.getOpcode() == AMDGPU::ENTER_STRICT_WWM ||236 MI.getOpcode() == AMDGPU::ENTER_STRICT_WQM) {237 LLVM_DEBUG(printWWMInfo(MI));238 InWWM = true;239 continue;240 }241 242 if (MI.getOpcode() == AMDGPU::EXIT_STRICT_WWM ||243 MI.getOpcode() == AMDGPU::EXIT_STRICT_WQM) {244 LLVM_DEBUG(printWWMInfo(MI));245 InWWM = false;246 }247 248 if (!InWWM)249 continue;250 251 LLVM_DEBUG(dbgs() << "Processing " << MI);252 253 for (MachineOperand &DefOpnd : MI.defs()) {254 RegsAssigned |= processDef(DefOpnd);255 }256 }257 }258 259 if (!RegsAssigned)260 return false;261 262 rewriteRegs(MF);263 return true;264}265 266PreservedAnalyses267SIPreAllocateWWMRegsPass::run(MachineFunction &MF,268 MachineFunctionAnalysisManager &MFAM) {269 auto *LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF);270 auto *Matrix = &MFAM.getResult<LiveRegMatrixAnalysis>(MF);271 auto *VRM = &MFAM.getResult<VirtRegMapAnalysis>(MF);272 SIPreAllocateWWMRegs(LIS, Matrix, VRM).run(MF);273 return PreservedAnalyses::all();274}275