532 lines · cpp
1//===- AMDGPUInsertDelayAlu.cpp - Insert s_delay_alu instructions ---------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// Insert s_delay_alu instructions to avoid stalls on GFX11+.11//12//===----------------------------------------------------------------------===//13 14#include "AMDGPU.h"15#include "GCNSubtarget.h"16#include "MCTargetDesc/AMDGPUMCTargetDesc.h"17#include "SIInstrInfo.h"18#include "llvm/ADT/SetVector.h"19 20using namespace llvm;21 22#define DEBUG_TYPE "amdgpu-insert-delay-alu"23 24namespace {25 26class AMDGPUInsertDelayAlu {27public:28 const GCNSubtarget *ST;29 const SIInstrInfo *SII;30 const TargetRegisterInfo *TRI;31 32 const TargetSchedModel *SchedModel;33 34 // Return true if MI waits for all outstanding VALU instructions to complete.35 static bool instructionWaitsForVALU(const MachineInstr &MI) {36 // These instruction types wait for VA_VDST==0 before issuing.37 const uint64_t VA_VDST_0 = SIInstrFlags::DS | SIInstrFlags::EXP |38 SIInstrFlags::FLAT | SIInstrFlags::MIMG |39 SIInstrFlags::MTBUF | SIInstrFlags::MUBUF;40 if (MI.getDesc().TSFlags & VA_VDST_0)41 return true;42 if (MI.getOpcode() == AMDGPU::S_SENDMSG_RTN_B32 ||43 MI.getOpcode() == AMDGPU::S_SENDMSG_RTN_B64)44 return true;45 if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&46 AMDGPU::DepCtr::decodeFieldVaVdst(MI.getOperand(0).getImm()) == 0)47 return true;48 return false;49 }50 51 static bool instructionWaitsForSGPRWrites(const MachineInstr &MI) {52 // These instruction types wait for VA_SDST==0 before issuing.53 uint64_t MIFlags = MI.getDesc().TSFlags;54 if (MIFlags & SIInstrFlags::SMRD)55 return true;56 57 if (MIFlags & SIInstrFlags::SALU) {58 for (auto &Op : MI.operands()) {59 if (Op.isReg())60 return true;61 }62 }63 return false;64 }65 66 // Types of delay that can be encoded in an s_delay_alu instruction.67 enum DelayType { VALU, TRANS, SALU, OTHER };68 69 // Get the delay type for a MachineInstr.70 DelayType getDelayType(const MachineInstr &MI) {71 if (SIInstrInfo::isTRANS(MI))72 return TRANS;73 // WMMA XDL ops are treated the same as TRANS.74 if (AMDGPU::isGFX1250(*ST) && SII->isXDLWMMA(MI))75 return TRANS;76 if (SIInstrInfo::isVALU(MI))77 return VALU;78 if (SIInstrInfo::isSALU(MI))79 return SALU;80 return OTHER;81 }82 83 // Information about the last instruction(s) that wrote to a particular84 // regunit. In straight-line code there will only be one such instruction, but85 // when control flow converges we merge the delay information from each path86 // to represent the union of the worst-case delays of each type.87 struct DelayInfo {88 // One larger than the maximum number of (non-TRANS) VALU instructions we89 // can encode in an s_delay_alu instruction.90 static constexpr unsigned VALU_MAX = 5;91 92 // One larger than the maximum number of TRANS instructions we can encode in93 // an s_delay_alu instruction.94 static constexpr unsigned TRANS_MAX = 4;95 96 // One larger than the maximum number of SALU cycles we can encode in an97 // s_delay_alu instruction.98 static constexpr unsigned SALU_CYCLES_MAX = 4;99 100 // If it was written by a (non-TRANS) VALU, remember how many clock cycles101 // are left until it completes, and how many other (non-TRANS) VALU we have102 // seen since it was issued.103 uint8_t VALUCycles = 0;104 uint8_t VALUNum = VALU_MAX;105 106 // If it was written by a TRANS, remember how many clock cycles are left107 // until it completes, and how many other TRANS we have seen since it was108 // issued.109 uint8_t TRANSCycles = 0;110 uint8_t TRANSNum = TRANS_MAX;111 // Also remember how many other (non-TRANS) VALU we have seen since it was112 // issued. When an instruction depends on both a prior TRANS and a prior113 // non-TRANS VALU, this is used to decide whether to encode a wait for just114 // one or both of them.115 uint8_t TRANSNumVALU = VALU_MAX;116 117 // If it was written by an SALU, remember how many clock cycles are left118 // until it completes.119 uint8_t SALUCycles = 0;120 121 DelayInfo() = default;122 123 DelayInfo(DelayType Type, unsigned Cycles) {124 switch (Type) {125 default:126 llvm_unreachable("unexpected type");127 case VALU:128 VALUCycles = Cycles;129 VALUNum = 0;130 break;131 case TRANS:132 TRANSCycles = Cycles;133 TRANSNum = 0;134 TRANSNumVALU = 0;135 break;136 case SALU:137 // Guard against pseudo-instructions like SI_CALL which are marked as138 // SALU but with a very high latency.139 SALUCycles = std::min(Cycles, SALU_CYCLES_MAX);140 break;141 }142 }143 144 bool operator==(const DelayInfo &RHS) const {145 return VALUCycles == RHS.VALUCycles && VALUNum == RHS.VALUNum &&146 TRANSCycles == RHS.TRANSCycles && TRANSNum == RHS.TRANSNum &&147 TRANSNumVALU == RHS.TRANSNumVALU && SALUCycles == RHS.SALUCycles;148 }149 150 bool operator!=(const DelayInfo &RHS) const { return !(*this == RHS); }151 152 // Merge another DelayInfo into this one, to represent the union of the153 // worst-case delays of each type.154 void merge(const DelayInfo &RHS) {155 VALUCycles = std::max(VALUCycles, RHS.VALUCycles);156 VALUNum = std::min(VALUNum, RHS.VALUNum);157 TRANSCycles = std::max(TRANSCycles, RHS.TRANSCycles);158 TRANSNum = std::min(TRANSNum, RHS.TRANSNum);159 TRANSNumVALU = std::min(TRANSNumVALU, RHS.TRANSNumVALU);160 SALUCycles = std::max(SALUCycles, RHS.SALUCycles);161 }162 163 // Update this DelayInfo after issuing an instruction of the specified type.164 // Cycles is the number of cycles it takes to issue the instruction. Return165 // true if there is no longer any useful delay info.166 bool advance(DelayType Type, unsigned Cycles) {167 bool Erase = true;168 169 VALUNum += (Type == VALU);170 if (VALUNum >= VALU_MAX || VALUCycles <= Cycles) {171 // Forget about the VALU instruction. It was too far back or has172 // definitely completed by now.173 VALUNum = VALU_MAX;174 VALUCycles = 0;175 } else {176 VALUCycles -= Cycles;177 Erase = false;178 }179 180 TRANSNum += (Type == TRANS);181 TRANSNumVALU += (Type == VALU);182 if (TRANSNum >= TRANS_MAX || TRANSCycles <= Cycles) {183 // Forget about any TRANS instruction. It was too far back or has184 // definitely completed by now.185 TRANSNum = TRANS_MAX;186 TRANSNumVALU = VALU_MAX;187 TRANSCycles = 0;188 } else {189 TRANSCycles -= Cycles;190 Erase = false;191 }192 193 if (SALUCycles <= Cycles) {194 // Forget about any SALU instruction. It has definitely completed by195 // now.196 SALUCycles = 0;197 } else {198 SALUCycles -= Cycles;199 Erase = false;200 }201 202 return Erase;203 }204 205#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)206 void dump() const {207 if (VALUCycles)208 dbgs() << " VALUCycles=" << (int)VALUCycles;209 if (VALUNum < VALU_MAX)210 dbgs() << " VALUNum=" << (int)VALUNum;211 if (TRANSCycles)212 dbgs() << " TRANSCycles=" << (int)TRANSCycles;213 if (TRANSNum < TRANS_MAX)214 dbgs() << " TRANSNum=" << (int)TRANSNum;215 if (TRANSNumVALU < VALU_MAX)216 dbgs() << " TRANSNumVALU=" << (int)TRANSNumVALU;217 if (SALUCycles)218 dbgs() << " SALUCycles=" << (int)SALUCycles;219 }220#endif221 };222 223 // A map from regunits to the delay info for that regunit.224 struct DelayState : DenseMap<MCRegUnit, DelayInfo> {225 // Merge another DelayState into this one by merging the delay info for each226 // regunit.227 void merge(const DelayState &RHS) {228 for (const auto &KV : RHS) {229 iterator It;230 bool Inserted;231 std::tie(It, Inserted) = insert(KV);232 if (!Inserted)233 It->second.merge(KV.second);234 }235 }236 237 // Advance the delay info for each regunit, erasing any that are no longer238 // useful.239 void advance(DelayType Type, unsigned Cycles) {240 iterator Next;241 for (auto I = begin(), E = end(); I != E; I = Next) {242 Next = std::next(I);243 if (I->second.advance(Type, Cycles))244 erase(I);245 }246 }247 248 void advanceByVALUNum(unsigned VALUNum) {249 iterator Next;250 for (auto I = begin(), E = end(); I != E; I = Next) {251 Next = std::next(I);252 if (I->second.VALUNum >= VALUNum && I->second.VALUCycles > 0) {253 erase(I);254 }255 }256 }257 258#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)259 void dump(const TargetRegisterInfo *TRI) const {260 if (empty()) {261 dbgs() << " empty\n";262 return;263 }264 265 // Dump DelayInfo for each RegUnit in numerical order.266 SmallVector<const_iterator, 8> Order;267 Order.reserve(size());268 for (const_iterator I = begin(), E = end(); I != E; ++I)269 Order.push_back(I);270 llvm::sort(Order, [](const const_iterator &A, const const_iterator &B) {271 return A->first < B->first;272 });273 for (const_iterator I : Order) {274 dbgs() << " " << printRegUnit(I->first, TRI);275 I->second.dump();276 dbgs() << "\n";277 }278 }279#endif280 };281 282 // The saved delay state at the end of each basic block.283 DenseMap<MachineBasicBlock *, DelayState> BlockState;284 285 // Emit an s_delay_alu instruction if necessary before MI.286 MachineInstr *emitDelayAlu(MachineInstr &MI, DelayInfo Delay,287 MachineInstr *LastDelayAlu) {288 unsigned Imm = 0;289 290 // Wait for a TRANS instruction.291 if (Delay.TRANSNum < DelayInfo::TRANS_MAX)292 Imm |= 4 + Delay.TRANSNum;293 294 // Wait for a VALU instruction (if it's more recent than any TRANS295 // instruction that we're also waiting for).296 if (Delay.VALUNum < DelayInfo::VALU_MAX &&297 Delay.VALUNum <= Delay.TRANSNumVALU) {298 if (Imm & 0xf)299 Imm |= Delay.VALUNum << 7;300 else301 Imm |= Delay.VALUNum;302 }303 304 // Wait for an SALU instruction.305 if (Delay.SALUCycles) {306 assert(Delay.SALUCycles < DelayInfo::SALU_CYCLES_MAX);307 if (Imm & 0x780) {308 // We have already encoded a VALU and a TRANS delay. There's no room in309 // the encoding for an SALU delay as well, so just drop it.310 } else if (Imm & 0xf) {311 Imm |= (Delay.SALUCycles + 8) << 7;312 } else {313 Imm |= Delay.SALUCycles + 8;314 }315 }316 317 // Don't emit the s_delay_alu instruction if there's nothing to wait for.318 if (!Imm)319 return LastDelayAlu;320 321 // If we only need to wait for one instruction, try encoding it in the last322 // s_delay_alu that we emitted.323 if (!(Imm & 0x780) && LastDelayAlu) {324 unsigned Skip = 0;325 for (auto I = MachineBasicBlock::instr_iterator(LastDelayAlu),326 E = MachineBasicBlock::instr_iterator(MI);327 ++I != E;) {328 if (!I->isBundle() && !I->isMetaInstruction())329 ++Skip;330 }331 if (Skip < 6) {332 MachineOperand &Op = LastDelayAlu->getOperand(0);333 unsigned LastImm = Op.getImm();334 assert((LastImm & ~0xf) == 0 &&335 "Remembered an s_delay_alu with no room for another delay!");336 LastImm |= Imm << 7 | Skip << 4;337 Op.setImm(LastImm);338 return nullptr;339 }340 }341 342 auto &MBB = *MI.getParent();343 MachineInstr *DelayAlu =344 BuildMI(MBB, MI, DebugLoc(), SII->get(AMDGPU::S_DELAY_ALU)).addImm(Imm);345 // Remember the s_delay_alu for next time if there is still room in it to346 // encode another delay.347 return (Imm & 0x780) ? nullptr : DelayAlu;348 }349 350 bool runOnMachineBasicBlock(MachineBasicBlock &MBB, bool Emit) {351 DelayState State;352 for (auto *Pred : MBB.predecessors())353 State.merge(BlockState[Pred]);354 355 LLVM_DEBUG(dbgs() << " State at start of " << printMBBReference(MBB)356 << "\n";357 State.dump(TRI););358 359 bool Changed = false;360 MachineInstr *LastDelayAlu = nullptr;361 362 // FIXME: 0 is a valid register unit.363 MCRegUnit LastSGPRFromVALU = static_cast<MCRegUnit>(0);364 // Iterate over the contents of bundles, but don't emit any instructions365 // inside a bundle.366 for (auto &MI : MBB.instrs()) {367 if (MI.isBundle() || MI.isMetaInstruction())368 continue;369 370 // Ignore some more instructions that do not generate any code.371 switch (MI.getOpcode()) {372 case AMDGPU::SI_RETURN_TO_EPILOG:373 continue;374 }375 376 DelayType Type = getDelayType(MI);377 378 if (instructionWaitsForSGPRWrites(MI)) {379 auto It = State.find(LastSGPRFromVALU);380 if (It != State.end()) {381 DelayInfo Info = It->getSecond();382 State.advanceByVALUNum(Info.VALUNum);383 // FIXME: 0 is a valid register unit.384 LastSGPRFromVALU = static_cast<MCRegUnit>(0);385 }386 }387 388 if (instructionWaitsForVALU(MI)) {389 // Forget about all outstanding VALU delays.390 // TODO: This is overkill since it also forgets about SALU delays.391 State = DelayState();392 } else if (Type != OTHER) {393 DelayInfo Delay;394 // TODO: Scan implicit uses too?395 for (const auto &Op : MI.explicit_uses()) {396 if (Op.isReg()) {397 // One of the operands of the writelane is also the output operand.398 // This creates the insertion of redundant delays. Hence, we have to399 // ignore this operand.400 if (MI.getOpcode() == AMDGPU::V_WRITELANE_B32 && Op.isTied())401 continue;402 for (MCRegUnit Unit : TRI->regunits(Op.getReg())) {403 auto It = State.find(Unit);404 if (It != State.end()) {405 Delay.merge(It->second);406 State.erase(Unit);407 }408 }409 }410 }411 412 if (SII->isVALU(MI.getOpcode())) {413 for (const auto &Op : MI.defs()) {414 Register Reg = Op.getReg();415 if (AMDGPU::isSGPR(Reg, TRI)) {416 LastSGPRFromVALU = *TRI->regunits(Reg).begin();417 break;418 }419 }420 }421 422 if (Emit && !MI.isBundledWithPred()) {423 // TODO: For VALU->SALU delays should we use s_delay_alu or s_nop or424 // just ignore them?425 LastDelayAlu = emitDelayAlu(MI, Delay, LastDelayAlu);426 }427 }428 429 if (Type != OTHER) {430 // TODO: Scan implicit defs too?431 for (const auto &Op : MI.defs()) {432 unsigned Latency = SchedModel->computeOperandLatency(433 &MI, Op.getOperandNo(), nullptr, 0);434 for (MCRegUnit Unit : TRI->regunits(Op.getReg()))435 State[Unit] = DelayInfo(Type, Latency);436 }437 }438 439 // Advance by the number of cycles it takes to issue this instruction.440 // TODO: Use a more advanced model that accounts for instructions that441 // take multiple cycles to issue on a particular pipeline.442 unsigned Cycles = SIInstrInfo::getNumWaitStates(MI);443 // TODO: In wave64 mode, double the number of cycles for VALU and VMEM444 // instructions on the assumption that they will usually have to be issued445 // twice?446 State.advance(Type, Cycles);447 448 LLVM_DEBUG(dbgs() << " State after " << MI; State.dump(TRI););449 }450 451 if (Emit) {452 assert(State == BlockState[&MBB] &&453 "Basic block state should not have changed on final pass!");454 } else if (DelayState &BS = BlockState[&MBB]; State != BS) {455 BS = std::move(State);456 Changed = true;457 }458 return Changed;459 }460 461 bool run(MachineFunction &MF) {462 LLVM_DEBUG(dbgs() << "AMDGPUInsertDelayAlu running on " << MF.getName()463 << "\n");464 465 ST = &MF.getSubtarget<GCNSubtarget>();466 if (!ST->hasDelayAlu())467 return false;468 469 SII = ST->getInstrInfo();470 TRI = ST->getRegisterInfo();471 SchedModel = &SII->getSchedModel();472 473 // Calculate the delay state for each basic block, iterating until we reach474 // a fixed point.475 SetVector<MachineBasicBlock *> WorkList;476 for (auto &MBB : reverse(MF))477 WorkList.insert(&MBB);478 while (!WorkList.empty()) {479 auto &MBB = *WorkList.pop_back_val();480 bool Changed = runOnMachineBasicBlock(MBB, false);481 if (Changed)482 WorkList.insert_range(MBB.successors());483 }484 485 LLVM_DEBUG(dbgs() << "Final pass over all BBs\n");486 487 // Make one last pass over all basic blocks to emit s_delay_alu488 // instructions.489 bool Changed = false;490 for (auto &MBB : MF)491 Changed |= runOnMachineBasicBlock(MBB, true);492 return Changed;493 }494};495 496class AMDGPUInsertDelayAluLegacy : public MachineFunctionPass {497public:498 static char ID;499 500 AMDGPUInsertDelayAluLegacy() : MachineFunctionPass(ID) {}501 502 void getAnalysisUsage(AnalysisUsage &AU) const override {503 AU.setPreservesCFG();504 MachineFunctionPass::getAnalysisUsage(AU);505 }506 507 bool runOnMachineFunction(MachineFunction &MF) override {508 if (skipFunction(MF.getFunction()))509 return false;510 AMDGPUInsertDelayAlu Impl;511 return Impl.run(MF);512 }513};514} // namespace515 516PreservedAnalyses517AMDGPUInsertDelayAluPass::run(MachineFunction &MF,518 MachineFunctionAnalysisManager &MFAM) {519 if (!AMDGPUInsertDelayAlu().run(MF))520 return PreservedAnalyses::all();521 auto PA = getMachineFunctionPassPreservedAnalyses();522 PA.preserveSet<CFGAnalyses>();523 return PA;524} // end namespace llvm525 526char AMDGPUInsertDelayAluLegacy::ID = 0;527 528char &llvm::AMDGPUInsertDelayAluID = AMDGPUInsertDelayAluLegacy::ID;529 530INITIALIZE_PASS(AMDGPUInsertDelayAluLegacy, DEBUG_TYPE,531 "AMDGPU Insert Delay ALU", false, false)532