1821 lines · cpp
1//===-- SIWholeQuadMode.cpp - enter and suspend whole quad mode -----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// This pass adds instructions to enable whole quad mode (strict or non-strict)11/// for pixel shaders, and strict whole wavefront mode for all programs.12///13/// The "strict" prefix indicates that inactive lanes do not take part in14/// control flow, specifically an inactive lane enabled by a strict WQM/WWM will15/// always be enabled irrespective of control flow decisions. Conversely in16/// non-strict WQM inactive lanes may control flow decisions.17///18/// Whole quad mode is required for derivative computations, but it interferes19/// with shader side effects (stores and atomics). It ensures that WQM is20/// enabled when necessary, but disabled around stores and atomics.21///22/// When necessary, this pass creates a function prolog23///24/// S_MOV_B64 LiveMask, EXEC25/// S_WQM_B64 EXEC, EXEC26///27/// to enter WQM at the top of the function and surrounds blocks of Exact28/// instructions by29///30/// S_AND_SAVEEXEC_B64 Tmp, LiveMask31/// ...32/// S_MOV_B64 EXEC, Tmp33///34/// We also compute when a sequence of instructions requires strict whole35/// wavefront mode (StrictWWM) and insert instructions to save and restore it:36///37/// S_OR_SAVEEXEC_B64 Tmp, -138/// ...39/// S_MOV_B64 EXEC, Tmp40///41/// When a sequence of instructions requires strict whole quad mode (StrictWQM)42/// we use a similar save and restore mechanism and force whole quad mode for43/// those instructions:44///45/// S_MOV_B64 Tmp, EXEC46/// S_WQM_B64 EXEC, EXEC47/// ...48/// S_MOV_B64 EXEC, Tmp49///50/// In order to avoid excessive switching during sequences of Exact51/// instructions, the pass first analyzes which instructions must be run in WQM52/// (aka which instructions produce values that lead to derivative53/// computations).54///55/// Basic blocks are always exited in WQM as long as some successor needs WQM.56///57/// There is room for improvement given better control flow analysis:58///59/// (1) at the top level (outside of control flow statements, and as long as60/// kill hasn't been used), one SGPR can be saved by recovering WQM from61/// the LiveMask (this is implemented for the entry block).62///63/// (2) when entire regions (e.g. if-else blocks or entire loops) only64/// consist of exact and don't-care instructions, the switch only has to65/// be done at the entry and exit points rather than potentially in each66/// block of the region.67///68//===----------------------------------------------------------------------===//69 70#include "SIWholeQuadMode.h"71#include "AMDGPU.h"72#include "AMDGPULaneMaskUtils.h"73#include "GCNSubtarget.h"74#include "MCTargetDesc/AMDGPUMCTargetDesc.h"75#include "llvm/ADT/MapVector.h"76#include "llvm/ADT/PostOrderIterator.h"77#include "llvm/CodeGen/LiveIntervals.h"78#include "llvm/CodeGen/MachineBasicBlock.h"79#include "llvm/CodeGen/MachineDominators.h"80#include "llvm/CodeGen/MachineFunctionPass.h"81#include "llvm/CodeGen/MachineInstr.h"82#include "llvm/CodeGen/MachinePostDominators.h"83#include "llvm/IR/CallingConv.h"84#include "llvm/InitializePasses.h"85#include "llvm/Support/raw_ostream.h"86 87using namespace llvm;88 89#define DEBUG_TYPE "si-wqm"90 91namespace {92 93enum {94 StateWQM = 0x1,95 StateStrictWWM = 0x2,96 StateStrictWQM = 0x4,97 StateExact = 0x8,98 StateStrict = StateStrictWWM | StateStrictWQM,99};100 101struct PrintState {102public:103 int State;104 105 explicit PrintState(int State) : State(State) {}106};107 108#ifndef NDEBUG109static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) {110 111 static const std::pair<char, const char *> Mapping[] = {112 std::pair(StateWQM, "WQM"), std::pair(StateStrictWWM, "StrictWWM"),113 std::pair(StateStrictWQM, "StrictWQM"), std::pair(StateExact, "Exact")};114 char State = PS.State;115 for (auto M : Mapping) {116 if (State & M.first) {117 OS << M.second;118 State &= ~M.first;119 120 if (State)121 OS << '|';122 }123 }124 assert(State == 0);125 return OS;126}127#endif128 129struct InstrInfo {130 char Needs = 0;131 char Disabled = 0;132 char OutNeeds = 0;133 char MarkedStates = 0;134};135 136struct BlockInfo {137 char Needs = 0;138 char InNeeds = 0;139 char OutNeeds = 0;140 char InitialState = 0;141 bool NeedsLowering = false;142};143 144struct WorkItem {145 MachineBasicBlock *MBB = nullptr;146 MachineInstr *MI = nullptr;147 148 WorkItem() = default;149 WorkItem(MachineBasicBlock *MBB) : MBB(MBB) {}150 WorkItem(MachineInstr *MI) : MI(MI) {}151};152 153class SIWholeQuadMode {154public:155 SIWholeQuadMode(MachineFunction &MF, LiveIntervals *LIS,156 MachineDominatorTree *MDT, MachinePostDominatorTree *PDT)157 : ST(&MF.getSubtarget<GCNSubtarget>()), TII(ST->getInstrInfo()),158 TRI(&TII->getRegisterInfo()), MRI(&MF.getRegInfo()), LIS(LIS), MDT(MDT),159 PDT(PDT), LMC(AMDGPU::LaneMaskConstants::get(*ST)) {}160 bool run(MachineFunction &MF);161 162private:163 const GCNSubtarget *ST;164 const SIInstrInfo *TII;165 const SIRegisterInfo *TRI;166 MachineRegisterInfo *MRI;167 LiveIntervals *LIS;168 MachineDominatorTree *MDT;169 MachinePostDominatorTree *PDT;170 const AMDGPU::LaneMaskConstants &LMC;171 172 Register LiveMaskReg;173 174 DenseMap<const MachineInstr *, InstrInfo> Instructions;175 MapVector<MachineBasicBlock *, BlockInfo> Blocks;176 177 // Tracks state (WQM/StrictWWM/StrictWQM/Exact) after a given instruction178 DenseMap<const MachineInstr *, char> StateTransition;179 180 SmallVector<MachineInstr *, 2> LiveMaskQueries;181 SmallVector<MachineInstr *, 4> LowerToMovInstrs;182 SmallSetVector<MachineInstr *, 4> LowerToCopyInstrs;183 SmallVector<MachineInstr *, 4> KillInstrs;184 SmallVector<MachineInstr *, 4> InitExecInstrs;185 SmallVector<MachineInstr *, 4> SetInactiveInstrs;186 187 void printInfo();188 189 void markInstruction(MachineInstr &MI, char Flag,190 std::vector<WorkItem> &Worklist);191 void markDefs(const MachineInstr &UseMI, LiveRange &LR,192 VirtRegOrUnit VRegOrUnit, unsigned SubReg, char Flag,193 std::vector<WorkItem> &Worklist);194 void markOperand(const MachineInstr &MI, const MachineOperand &Op, char Flag,195 std::vector<WorkItem> &Worklist);196 void markInstructionUses(const MachineInstr &MI, char Flag,197 std::vector<WorkItem> &Worklist);198 char scanInstructions(MachineFunction &MF, std::vector<WorkItem> &Worklist);199 void propagateInstruction(MachineInstr &MI, std::vector<WorkItem> &Worklist);200 void propagateBlock(MachineBasicBlock &MBB, std::vector<WorkItem> &Worklist);201 char analyzeFunction(MachineFunction &MF);202 203 MachineBasicBlock::iterator saveSCC(MachineBasicBlock &MBB,204 MachineBasicBlock::iterator Before);205 MachineBasicBlock::iterator206 prepareInsertion(MachineBasicBlock &MBB, MachineBasicBlock::iterator First,207 MachineBasicBlock::iterator Last, bool PreferLast,208 bool SaveSCC);209 void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,210 Register SaveWQM);211 void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,212 Register SavedWQM);213 void toStrictMode(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,214 Register SaveOrig, char StrictStateNeeded);215 void fromStrictMode(MachineBasicBlock &MBB,216 MachineBasicBlock::iterator Before, Register SavedOrig,217 char NonStrictState, char CurrentStrictState);218 219 void splitBlock(MachineInstr *TermMI);220 MachineInstr *lowerKillI1(MachineInstr &MI, bool IsWQM);221 MachineInstr *lowerKillF32(MachineInstr &MI);222 223 void lowerBlock(MachineBasicBlock &MBB, BlockInfo &BI);224 void processBlock(MachineBasicBlock &MBB, BlockInfo &BI, bool IsEntry);225 226 bool lowerLiveMaskQueries();227 bool lowerCopyInstrs();228 bool lowerKillInstrs(bool IsWQM);229 void lowerInitExec(MachineInstr &MI);230 MachineBasicBlock::iterator lowerInitExecInstrs(MachineBasicBlock &Entry,231 bool &Changed);232};233 234class SIWholeQuadModeLegacy : public MachineFunctionPass {235public:236 static char ID;237 238 SIWholeQuadModeLegacy() : MachineFunctionPass(ID) {}239 240 bool runOnMachineFunction(MachineFunction &MF) override;241 242 StringRef getPassName() const override { return "SI Whole Quad Mode"; }243 244 void getAnalysisUsage(AnalysisUsage &AU) const override {245 AU.addRequired<LiveIntervalsWrapperPass>();246 AU.addPreserved<SlotIndexesWrapperPass>();247 AU.addPreserved<LiveIntervalsWrapperPass>();248 AU.addPreserved<MachineDominatorTreeWrapperPass>();249 AU.addPreserved<MachinePostDominatorTreeWrapperPass>();250 MachineFunctionPass::getAnalysisUsage(AU);251 }252 253 MachineFunctionProperties getClearedProperties() const override {254 return MachineFunctionProperties().setIsSSA();255 }256};257} // end anonymous namespace258 259char SIWholeQuadModeLegacy::ID = 0;260 261INITIALIZE_PASS_BEGIN(SIWholeQuadModeLegacy, DEBUG_TYPE, "SI Whole Quad Mode",262 false, false)263INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)264INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)265INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTreeWrapperPass)266INITIALIZE_PASS_END(SIWholeQuadModeLegacy, DEBUG_TYPE, "SI Whole Quad Mode",267 false, false)268 269char &llvm::SIWholeQuadModeID = SIWholeQuadModeLegacy::ID;270 271FunctionPass *llvm::createSIWholeQuadModeLegacyPass() {272 return new SIWholeQuadModeLegacy;273}274 275#ifndef NDEBUG276LLVM_DUMP_METHOD void SIWholeQuadMode::printInfo() {277 for (const auto &BII : Blocks) {278 dbgs() << "\n"279 << printMBBReference(*BII.first) << ":\n"280 << " InNeeds = " << PrintState(BII.second.InNeeds)281 << ", Needs = " << PrintState(BII.second.Needs)282 << ", OutNeeds = " << PrintState(BII.second.OutNeeds) << "\n\n";283 284 for (const MachineInstr &MI : *BII.first) {285 auto III = Instructions.find(&MI);286 if (III != Instructions.end()) {287 dbgs() << " " << MI << " Needs = " << PrintState(III->second.Needs)288 << ", OutNeeds = " << PrintState(III->second.OutNeeds) << '\n';289 }290 }291 }292}293#endif294 295void SIWholeQuadMode::markInstruction(MachineInstr &MI, char Flag,296 std::vector<WorkItem> &Worklist) {297 InstrInfo &II = Instructions[&MI];298 299 assert(!(Flag & StateExact) && Flag != 0);300 301 // Capture all states requested in marking including disabled ones.302 II.MarkedStates |= Flag;303 304 // Remove any disabled states from the flag. The user that required it gets305 // an undefined value in the helper lanes. For example, this can happen if306 // the result of an atomic is used by instruction that requires WQM, where307 // ignoring the request for WQM is correct as per the relevant specs.308 Flag &= ~II.Disabled;309 310 // Ignore if the flag is already encompassed by the existing needs, or we311 // just disabled everything.312 if ((II.Needs & Flag) == Flag)313 return;314 315 LLVM_DEBUG(dbgs() << "markInstruction " << PrintState(Flag) << ": " << MI);316 II.Needs |= Flag;317 Worklist.emplace_back(&MI);318}319 320/// Mark all relevant definitions of register \p Reg in usage \p UseMI.321void SIWholeQuadMode::markDefs(const MachineInstr &UseMI, LiveRange &LR,322 VirtRegOrUnit VRegOrUnit, unsigned SubReg,323 char Flag, std::vector<WorkItem> &Worklist) {324 LLVM_DEBUG(dbgs() << "markDefs " << PrintState(Flag) << ": " << UseMI);325 326 LiveQueryResult UseLRQ = LR.Query(LIS->getInstructionIndex(UseMI));327 const VNInfo *Value = UseLRQ.valueIn();328 if (!Value)329 return;330 331 // Note: this code assumes that lane masks on AMDGPU completely332 // cover registers.333 const LaneBitmask UseLanes =334 SubReg ? TRI->getSubRegIndexLaneMask(SubReg)335 : (VRegOrUnit.isVirtualReg()336 ? MRI->getMaxLaneMaskForVReg(VRegOrUnit.asVirtualReg())337 : LaneBitmask::getNone());338 339 // Perform a depth-first iteration of the LiveRange graph marking defs.340 // Stop processing of a given branch when all use lanes have been defined.341 // The first definition stops processing for a physical register.342 struct PhiEntry {343 const VNInfo *Phi;344 unsigned PredIdx;345 LaneBitmask DefinedLanes;346 347 PhiEntry(const VNInfo *Phi, unsigned PredIdx, LaneBitmask DefinedLanes)348 : Phi(Phi), PredIdx(PredIdx), DefinedLanes(DefinedLanes) {}349 };350 using VisitKey = std::pair<const VNInfo *, LaneBitmask>;351 SmallVector<PhiEntry, 2> PhiStack;352 SmallSet<VisitKey, 4> Visited;353 LaneBitmask DefinedLanes;354 unsigned NextPredIdx = 0; // Only used for processing phi nodes355 do {356 const VNInfo *NextValue = nullptr;357 const VisitKey Key(Value, DefinedLanes);358 359 if (Visited.insert(Key).second) {360 // On first visit to a phi then start processing first predecessor361 NextPredIdx = 0;362 }363 364 if (Value->isPHIDef()) {365 // Each predecessor node in the phi must be processed as a subgraph366 const MachineBasicBlock *MBB = LIS->getMBBFromIndex(Value->def);367 assert(MBB && "Phi-def has no defining MBB");368 369 // Find next predecessor to process370 unsigned Idx = NextPredIdx;371 const auto *PI = MBB->pred_begin() + Idx;372 const auto *PE = MBB->pred_end();373 for (; PI != PE && !NextValue; ++PI, ++Idx) {374 if (const VNInfo *VN = LR.getVNInfoBefore(LIS->getMBBEndIdx(*PI))) {375 if (!Visited.count(VisitKey(VN, DefinedLanes)))376 NextValue = VN;377 }378 }379 380 // If there are more predecessors to process; add phi to stack381 if (PI != PE)382 PhiStack.emplace_back(Value, Idx, DefinedLanes);383 } else {384 MachineInstr *MI = LIS->getInstructionFromIndex(Value->def);385 assert(MI && "Def has no defining instruction");386 387 if (VRegOrUnit.isVirtualReg()) {388 // Iterate over all operands to find relevant definitions389 bool HasDef = false;390 for (const MachineOperand &Op : MI->all_defs()) {391 if (Op.getReg() != VRegOrUnit.asVirtualReg())392 continue;393 394 // Compute lanes defined and overlap with use395 LaneBitmask OpLanes =396 Op.isUndef() ? LaneBitmask::getAll()397 : TRI->getSubRegIndexLaneMask(Op.getSubReg());398 LaneBitmask Overlap = (UseLanes & OpLanes);399 400 // Record if this instruction defined any of use401 HasDef |= Overlap.any();402 403 // Mark any lanes defined404 DefinedLanes |= OpLanes;405 }406 407 // Check if all lanes of use have been defined408 if ((DefinedLanes & UseLanes) != UseLanes) {409 // Definition not complete; need to process input value410 LiveQueryResult LRQ = LR.Query(LIS->getInstructionIndex(*MI));411 if (const VNInfo *VN = LRQ.valueIn()) {412 if (!Visited.count(VisitKey(VN, DefinedLanes)))413 NextValue = VN;414 }415 }416 417 // Only mark the instruction if it defines some part of the use418 if (HasDef)419 markInstruction(*MI, Flag, Worklist);420 } else {421 // For physical registers simply mark the defining instruction422 markInstruction(*MI, Flag, Worklist);423 }424 }425 426 if (!NextValue && !PhiStack.empty()) {427 // Reach end of chain; revert to processing last phi428 PhiEntry &Entry = PhiStack.back();429 NextValue = Entry.Phi;430 NextPredIdx = Entry.PredIdx;431 DefinedLanes = Entry.DefinedLanes;432 PhiStack.pop_back();433 }434 435 Value = NextValue;436 } while (Value);437}438 439void SIWholeQuadMode::markOperand(const MachineInstr &MI,440 const MachineOperand &Op, char Flag,441 std::vector<WorkItem> &Worklist) {442 assert(Op.isReg());443 Register Reg = Op.getReg();444 445 // Ignore some hardware registers446 switch (Reg) {447 case AMDGPU::EXEC:448 case AMDGPU::EXEC_LO:449 return;450 default:451 break;452 }453 454 LLVM_DEBUG(dbgs() << "markOperand " << PrintState(Flag) << ": " << Op455 << " for " << MI);456 if (Reg.isVirtual()) {457 LiveRange &LR = LIS->getInterval(Reg);458 markDefs(MI, LR, VirtRegOrUnit(Reg), Op.getSubReg(), Flag, Worklist);459 } else {460 // Handle physical registers that we need to track; this is mostly relevant461 // for VCC, which can appear as the (implicit) input of a uniform branch,462 // e.g. when a loop counter is stored in a VGPR.463 for (MCRegUnit Unit : TRI->regunits(Reg.asMCReg())) {464 LiveRange &LR = LIS->getRegUnit(Unit);465 const VNInfo *Value = LR.Query(LIS->getInstructionIndex(MI)).valueIn();466 if (Value)467 markDefs(MI, LR, VirtRegOrUnit(Unit), AMDGPU::NoSubRegister, Flag,468 Worklist);469 }470 }471}472 473/// Mark all instructions defining the uses in \p MI with \p Flag.474void SIWholeQuadMode::markInstructionUses(const MachineInstr &MI, char Flag,475 std::vector<WorkItem> &Worklist) {476 LLVM_DEBUG(dbgs() << "markInstructionUses " << PrintState(Flag) << ": "477 << MI);478 479 for (const MachineOperand &Use : MI.all_uses())480 markOperand(MI, Use, Flag, Worklist);481}482 483// Scan instructions to determine which ones require an Exact execmask and484// which ones seed WQM requirements.485char SIWholeQuadMode::scanInstructions(MachineFunction &MF,486 std::vector<WorkItem> &Worklist) {487 char GlobalFlags = 0;488 bool WQMOutputs = MF.getFunction().hasFnAttribute("amdgpu-ps-wqm-outputs");489 SmallVector<MachineInstr *, 4> SoftWQMInstrs;490 bool HasImplicitDerivatives =491 MF.getFunction().getCallingConv() == CallingConv::AMDGPU_PS;492 493 // We need to visit the basic blocks in reverse post-order so that we visit494 // defs before uses, in particular so that we don't accidentally mark an495 // instruction as needing e.g. WQM before visiting it and realizing it needs496 // WQM disabled.497 ReversePostOrderTraversal<MachineFunction *> RPOT(&MF);498 for (MachineBasicBlock *MBB : RPOT) {499 BlockInfo &BBI = Blocks[MBB];500 501 for (MachineInstr &MI : *MBB) {502 InstrInfo &III = Instructions[&MI];503 unsigned Opcode = MI.getOpcode();504 char Flags = 0;505 506 if (TII->isWQM(Opcode)) {507 // If LOD is not supported WQM is not needed.508 // Only generate implicit WQM if implicit derivatives are required.509 // This avoids inserting unintended WQM if a shader type without510 // implicit derivatives uses an image sampling instruction.511 if (ST->hasExtendedImageInsts() && HasImplicitDerivatives) {512 // Sampling instructions don't need to produce results for all pixels513 // in a quad, they just require all inputs of a quad to have been514 // computed for derivatives.515 markInstructionUses(MI, StateWQM, Worklist);516 GlobalFlags |= StateWQM;517 }518 } else if (Opcode == AMDGPU::WQM) {519 // The WQM intrinsic requires its output to have all the helper lanes520 // correct, so we need it to be in WQM.521 Flags = StateWQM;522 LowerToCopyInstrs.insert(&MI);523 } else if (Opcode == AMDGPU::SOFT_WQM) {524 LowerToCopyInstrs.insert(&MI);525 SoftWQMInstrs.push_back(&MI);526 } else if (Opcode == AMDGPU::STRICT_WWM) {527 // The STRICT_WWM intrinsic doesn't make the same guarantee, and plus528 // it needs to be executed in WQM or Exact so that its copy doesn't529 // clobber inactive lanes.530 markInstructionUses(MI, StateStrictWWM, Worklist);531 GlobalFlags |= StateStrictWWM;532 LowerToMovInstrs.push_back(&MI);533 } else if (Opcode == AMDGPU::STRICT_WQM ||534 TII->isDualSourceBlendEXP(MI)) {535 // STRICT_WQM is similar to STRICTWWM, but instead of enabling all536 // threads of the wave like STRICTWWM, STRICT_WQM enables all threads in537 // quads that have at least one active thread.538 markInstructionUses(MI, StateStrictWQM, Worklist);539 GlobalFlags |= StateStrictWQM;540 541 if (Opcode == AMDGPU::STRICT_WQM) {542 LowerToMovInstrs.push_back(&MI);543 } else {544 // Dual source blend export acts as implicit strict-wqm, its sources545 // need to be shuffled in strict wqm, but the export itself needs to546 // run in exact mode.547 BBI.Needs |= StateExact;548 if (!(BBI.InNeeds & StateExact)) {549 BBI.InNeeds |= StateExact;550 Worklist.emplace_back(MBB);551 }552 GlobalFlags |= StateExact;553 III.Disabled = StateWQM | StateStrict;554 }555 } else if (Opcode == AMDGPU::LDS_PARAM_LOAD ||556 Opcode == AMDGPU::DS_PARAM_LOAD ||557 Opcode == AMDGPU::LDS_DIRECT_LOAD ||558 Opcode == AMDGPU::DS_DIRECT_LOAD) {559 // Mark these STRICTWQM, but only for the instruction, not its operands.560 // This avoid unnecessarily marking M0 as requiring WQM.561 III.Needs |= StateStrictWQM;562 GlobalFlags |= StateStrictWQM;563 } else if (Opcode == AMDGPU::V_SET_INACTIVE_B32) {564 // Disable strict states; StrictWQM will be added as required later.565 III.Disabled = StateStrict;566 MachineOperand &Inactive = MI.getOperand(4);567 if (Inactive.isReg()) {568 if (Inactive.isUndef() && MI.getOperand(3).getImm() == 0)569 LowerToCopyInstrs.insert(&MI);570 else571 markOperand(MI, Inactive, StateStrictWWM, Worklist);572 }573 SetInactiveInstrs.push_back(&MI);574 BBI.NeedsLowering = true;575 } else if (TII->isDisableWQM(MI)) {576 BBI.Needs |= StateExact;577 if (!(BBI.InNeeds & StateExact)) {578 BBI.InNeeds |= StateExact;579 Worklist.emplace_back(MBB);580 }581 GlobalFlags |= StateExact;582 III.Disabled = StateWQM | StateStrict;583 } else if (Opcode == AMDGPU::SI_PS_LIVE ||584 Opcode == AMDGPU::SI_LIVE_MASK) {585 LiveMaskQueries.push_back(&MI);586 } else if (Opcode == AMDGPU::SI_KILL_I1_TERMINATOR ||587 Opcode == AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR ||588 Opcode == AMDGPU::SI_DEMOTE_I1) {589 KillInstrs.push_back(&MI);590 BBI.NeedsLowering = true;591 } else if (Opcode == AMDGPU::SI_INIT_EXEC ||592 Opcode == AMDGPU::SI_INIT_EXEC_FROM_INPUT ||593 Opcode == AMDGPU::SI_INIT_WHOLE_WAVE) {594 InitExecInstrs.push_back(&MI);595 } else if (WQMOutputs) {596 // The function is in machine SSA form, which means that physical597 // VGPRs correspond to shader inputs and outputs. Inputs are598 // only used, outputs are only defined.599 // FIXME: is this still valid?600 for (const MachineOperand &MO : MI.defs()) {601 Register Reg = MO.getReg();602 if (Reg.isPhysical() &&603 TRI->hasVectorRegisters(TRI->getPhysRegBaseClass(Reg))) {604 Flags = StateWQM;605 break;606 }607 }608 }609 610 if (Flags) {611 markInstruction(MI, Flags, Worklist);612 GlobalFlags |= Flags;613 }614 }615 }616 617 // Mark sure that any SET_INACTIVE instructions are computed in WQM if WQM is618 // ever used anywhere in the function. This implements the corresponding619 // semantics of @llvm.amdgcn.set.inactive.620 // Similarly for SOFT_WQM instructions, implementing @llvm.amdgcn.softwqm.621 if (GlobalFlags & StateWQM) {622 for (MachineInstr *MI : SetInactiveInstrs)623 markInstruction(*MI, StateWQM, Worklist);624 for (MachineInstr *MI : SoftWQMInstrs)625 markInstruction(*MI, StateWQM, Worklist);626 }627 628 return GlobalFlags;629}630 631void SIWholeQuadMode::propagateInstruction(MachineInstr &MI,632 std::vector<WorkItem>& Worklist) {633 MachineBasicBlock *MBB = MI.getParent();634 InstrInfo II = Instructions[&MI]; // take a copy to prevent dangling references635 BlockInfo &BI = Blocks[MBB];636 637 // Control flow-type instructions and stores to temporary memory that are638 // followed by WQM computations must themselves be in WQM.639 if ((II.OutNeeds & StateWQM) && !(II.Disabled & StateWQM) &&640 (MI.isTerminator() || (TII->usesVM_CNT(MI) && MI.mayStore()))) {641 Instructions[&MI].Needs = StateWQM;642 II.Needs = StateWQM;643 }644 645 // Propagate to block level646 if (II.Needs & StateWQM) {647 BI.Needs |= StateWQM;648 if (!(BI.InNeeds & StateWQM)) {649 BI.InNeeds |= StateWQM;650 Worklist.emplace_back(MBB);651 }652 }653 654 // Propagate backwards within block655 if (MachineInstr *PrevMI = MI.getPrevNode()) {656 char InNeeds = (II.Needs & ~StateStrict) | II.OutNeeds;657 if (!PrevMI->isPHI()) {658 InstrInfo &PrevII = Instructions[PrevMI];659 if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {660 PrevII.OutNeeds |= InNeeds;661 Worklist.emplace_back(PrevMI);662 }663 }664 }665 666 // Propagate WQM flag to instruction inputs667 assert(!(II.Needs & StateExact));668 669 if (II.Needs != 0)670 markInstructionUses(MI, II.Needs, Worklist);671 672 // Ensure we process a block containing StrictWWM/StrictWQM, even if it does673 // not require any WQM transitions.674 if (II.Needs & StateStrictWWM)675 BI.Needs |= StateStrictWWM;676 if (II.Needs & StateStrictWQM)677 BI.Needs |= StateStrictWQM;678}679 680void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB,681 std::vector<WorkItem>& Worklist) {682 BlockInfo BI = Blocks[&MBB]; // Make a copy to prevent dangling references.683 684 // Propagate through instructions685 if (!MBB.empty()) {686 MachineInstr *LastMI = &*MBB.rbegin();687 InstrInfo &LastII = Instructions[LastMI];688 if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) {689 LastII.OutNeeds |= BI.OutNeeds;690 Worklist.emplace_back(LastMI);691 }692 }693 694 // Predecessor blocks must provide for our WQM/Exact needs.695 for (MachineBasicBlock *Pred : MBB.predecessors()) {696 BlockInfo &PredBI = Blocks[Pred];697 if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds)698 continue;699 700 PredBI.OutNeeds |= BI.InNeeds;701 PredBI.InNeeds |= BI.InNeeds;702 Worklist.emplace_back(Pred);703 }704 705 // All successors must be prepared to accept the same set of WQM/Exact data.706 for (MachineBasicBlock *Succ : MBB.successors()) {707 BlockInfo &SuccBI = Blocks[Succ];708 if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds)709 continue;710 711 SuccBI.InNeeds |= BI.OutNeeds;712 Worklist.emplace_back(Succ);713 }714}715 716char SIWholeQuadMode::analyzeFunction(MachineFunction &MF) {717 std::vector<WorkItem> Worklist;718 char GlobalFlags = scanInstructions(MF, Worklist);719 720 while (!Worklist.empty()) {721 WorkItem WI = Worklist.back();722 Worklist.pop_back();723 724 if (WI.MI)725 propagateInstruction(*WI.MI, Worklist);726 else727 propagateBlock(*WI.MBB, Worklist);728 }729 730 return GlobalFlags;731}732 733MachineBasicBlock::iterator734SIWholeQuadMode::saveSCC(MachineBasicBlock &MBB,735 MachineBasicBlock::iterator Before) {736 Register SaveReg = MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);737 738 MachineInstr *Save =739 BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), SaveReg)740 .addReg(AMDGPU::SCC);741 MachineInstr *Restore =742 BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), AMDGPU::SCC)743 .addReg(SaveReg);744 745 LIS->InsertMachineInstrInMaps(*Save);746 LIS->InsertMachineInstrInMaps(*Restore);747 LIS->createAndComputeVirtRegInterval(SaveReg);748 749 return Restore;750}751 752void SIWholeQuadMode::splitBlock(MachineInstr *TermMI) {753 MachineBasicBlock *BB = TermMI->getParent();754 LLVM_DEBUG(dbgs() << "Split block " << printMBBReference(*BB) << " @ "755 << *TermMI << "\n");756 757 MachineBasicBlock *SplitBB =758 BB->splitAt(*TermMI, /*UpdateLiveIns*/ true, LIS);759 760 // Convert last instruction in block to a terminator.761 // Note: this only covers the expected patterns762 unsigned NewOpcode = 0;763 switch (TermMI->getOpcode()) {764 case AMDGPU::S_AND_B32:765 NewOpcode = AMDGPU::S_AND_B32_term;766 break;767 case AMDGPU::S_AND_B64:768 NewOpcode = AMDGPU::S_AND_B64_term;769 break;770 case AMDGPU::S_MOV_B32:771 NewOpcode = AMDGPU::S_MOV_B32_term;772 break;773 case AMDGPU::S_MOV_B64:774 NewOpcode = AMDGPU::S_MOV_B64_term;775 break;776 case AMDGPU::S_ANDN2_B32:777 NewOpcode = AMDGPU::S_ANDN2_B32_term;778 break;779 case AMDGPU::S_ANDN2_B64:780 NewOpcode = AMDGPU::S_ANDN2_B64_term;781 break;782 default:783 llvm_unreachable("Unexpected instruction");784 }785 786 // These terminators fallthrough to the next block, no need to add an787 // unconditional branch to the next block (SplitBB).788 TermMI->setDesc(TII->get(NewOpcode));789 790 if (SplitBB != BB) {791 // Update dominator trees792 using DomTreeT = DomTreeBase<MachineBasicBlock>;793 SmallVector<DomTreeT::UpdateType, 16> DTUpdates;794 for (MachineBasicBlock *Succ : SplitBB->successors()) {795 DTUpdates.push_back({DomTreeT::Insert, SplitBB, Succ});796 DTUpdates.push_back({DomTreeT::Delete, BB, Succ});797 }798 DTUpdates.push_back({DomTreeT::Insert, BB, SplitBB});799 if (MDT)800 MDT->applyUpdates(DTUpdates);801 if (PDT)802 PDT->applyUpdates(DTUpdates);803 }804}805 806MachineInstr *SIWholeQuadMode::lowerKillF32(MachineInstr &MI) {807 assert(LiveMaskReg.isVirtual());808 809 const DebugLoc &DL = MI.getDebugLoc();810 unsigned Opcode = 0;811 812 assert(MI.getOperand(0).isReg());813 814 // Comparison is for live lanes; however here we compute the inverse815 // (killed lanes). This is because VCMP will always generate 0 bits816 // for inactive lanes so a mask of live lanes would not be correct817 // inside control flow.818 // Invert the comparison by swapping the operands and adjusting819 // the comparison codes.820 821 switch (MI.getOperand(2).getImm()) {822 case ISD::SETUEQ:823 Opcode = AMDGPU::V_CMP_LG_F32_e64;824 break;825 case ISD::SETUGT:826 Opcode = AMDGPU::V_CMP_GE_F32_e64;827 break;828 case ISD::SETUGE:829 Opcode = AMDGPU::V_CMP_GT_F32_e64;830 break;831 case ISD::SETULT:832 Opcode = AMDGPU::V_CMP_LE_F32_e64;833 break;834 case ISD::SETULE:835 Opcode = AMDGPU::V_CMP_LT_F32_e64;836 break;837 case ISD::SETUNE:838 Opcode = AMDGPU::V_CMP_EQ_F32_e64;839 break;840 case ISD::SETO:841 Opcode = AMDGPU::V_CMP_O_F32_e64;842 break;843 case ISD::SETUO:844 Opcode = AMDGPU::V_CMP_U_F32_e64;845 break;846 case ISD::SETOEQ:847 case ISD::SETEQ:848 Opcode = AMDGPU::V_CMP_NEQ_F32_e64;849 break;850 case ISD::SETOGT:851 case ISD::SETGT:852 Opcode = AMDGPU::V_CMP_NLT_F32_e64;853 break;854 case ISD::SETOGE:855 case ISD::SETGE:856 Opcode = AMDGPU::V_CMP_NLE_F32_e64;857 break;858 case ISD::SETOLT:859 case ISD::SETLT:860 Opcode = AMDGPU::V_CMP_NGT_F32_e64;861 break;862 case ISD::SETOLE:863 case ISD::SETLE:864 Opcode = AMDGPU::V_CMP_NGE_F32_e64;865 break;866 case ISD::SETONE:867 case ISD::SETNE:868 Opcode = AMDGPU::V_CMP_NLG_F32_e64;869 break;870 default:871 llvm_unreachable("invalid ISD:SET cond code");872 }873 874 MachineBasicBlock &MBB = *MI.getParent();875 876 // Pick opcode based on comparison type.877 MachineInstr *VcmpMI;878 const MachineOperand &Op0 = MI.getOperand(0);879 const MachineOperand &Op1 = MI.getOperand(1);880 881 // VCC represents lanes killed.882 if (TRI->isVGPR(*MRI, Op0.getReg())) {883 Opcode = AMDGPU::getVOPe32(Opcode);884 VcmpMI = BuildMI(MBB, &MI, DL, TII->get(Opcode)).add(Op1).add(Op0);885 } else {886 VcmpMI = BuildMI(MBB, &MI, DL, TII->get(Opcode))887 .addReg(LMC.VccReg, RegState::Define)888 .addImm(0) // src0 modifiers889 .add(Op1)890 .addImm(0) // src1 modifiers891 .add(Op0)892 .addImm(0); // omod893 }894 895 MachineInstr *MaskUpdateMI =896 BuildMI(MBB, MI, DL, TII->get(LMC.AndN2Opc), LiveMaskReg)897 .addReg(LiveMaskReg)898 .addReg(LMC.VccReg);899 900 // State of SCC represents whether any lanes are live in mask,901 // if SCC is 0 then no lanes will be alive anymore.902 MachineInstr *EarlyTermMI =903 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_EARLY_TERMINATE_SCC0));904 905 MachineInstr *ExecMaskMI =906 BuildMI(MBB, MI, DL, TII->get(LMC.AndN2Opc), LMC.ExecReg)907 .addReg(LMC.ExecReg)908 .addReg(LMC.VccReg);909 910 assert(MBB.succ_size() == 1);911 912 // Update live intervals913 LIS->ReplaceMachineInstrInMaps(MI, *VcmpMI);914 MBB.remove(&MI);915 916 LIS->InsertMachineInstrInMaps(*MaskUpdateMI);917 LIS->InsertMachineInstrInMaps(*EarlyTermMI);918 LIS->InsertMachineInstrInMaps(*ExecMaskMI);919 920 return ExecMaskMI;921}922 923MachineInstr *SIWholeQuadMode::lowerKillI1(MachineInstr &MI, bool IsWQM) {924 assert(LiveMaskReg.isVirtual());925 926 MachineBasicBlock &MBB = *MI.getParent();927 928 const DebugLoc &DL = MI.getDebugLoc();929 MachineInstr *MaskUpdateMI = nullptr;930 931 const bool IsDemote = IsWQM && (MI.getOpcode() == AMDGPU::SI_DEMOTE_I1);932 const MachineOperand &Op = MI.getOperand(0);933 int64_t KillVal = MI.getOperand(1).getImm();934 MachineInstr *ComputeKilledMaskMI = nullptr;935 Register CndReg = !Op.isImm() ? Op.getReg() : Register();936 Register TmpReg;937 938 // Is this a static or dynamic kill?939 if (Op.isImm()) {940 if (Op.getImm() == KillVal) {941 // Static: all active lanes are killed942 MaskUpdateMI = BuildMI(MBB, MI, DL, TII->get(LMC.AndN2Opc), LiveMaskReg)943 .addReg(LiveMaskReg)944 .addReg(LMC.ExecReg);945 } else {946 // Static: kill does nothing947 bool IsLastTerminator = std::next(MI.getIterator()) == MBB.end();948 if (!IsLastTerminator) {949 LIS->RemoveMachineInstrFromMaps(MI);950 } else {951 assert(MBB.succ_size() == 1 && MI.getOpcode() != AMDGPU::SI_DEMOTE_I1);952 MachineInstr *NewTerm = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_BRANCH))953 .addMBB(*MBB.succ_begin());954 LIS->ReplaceMachineInstrInMaps(MI, *NewTerm);955 }956 MBB.remove(&MI);957 return nullptr;958 }959 } else {960 if (!KillVal) {961 // Op represents live lanes after kill,962 // so exec mask needs to be factored in.963 TmpReg = MRI->createVirtualRegister(TRI->getBoolRC());964 ComputeKilledMaskMI = BuildMI(MBB, MI, DL, TII->get(LMC.AndN2Opc), TmpReg)965 .addReg(LMC.ExecReg)966 .add(Op);967 MaskUpdateMI = BuildMI(MBB, MI, DL, TII->get(LMC.AndN2Opc), LiveMaskReg)968 .addReg(LiveMaskReg)969 .addReg(TmpReg);970 } else {971 // Op represents lanes to kill972 MaskUpdateMI = BuildMI(MBB, MI, DL, TII->get(LMC.AndN2Opc), LiveMaskReg)973 .addReg(LiveMaskReg)974 .add(Op);975 }976 }977 978 // State of SCC represents whether any lanes are live in mask,979 // if SCC is 0 then no lanes will be alive anymore.980 MachineInstr *EarlyTermMI =981 BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_EARLY_TERMINATE_SCC0));982 983 // In the case we got this far some lanes are still live,984 // update EXEC to deactivate lanes as appropriate.985 MachineInstr *NewTerm;986 MachineInstr *WQMMaskMI = nullptr;987 Register LiveMaskWQM;988 if (IsDemote) {989 // Demote - deactivate quads with only helper lanes990 LiveMaskWQM = MRI->createVirtualRegister(TRI->getBoolRC());991 WQMMaskMI = BuildMI(MBB, MI, DL, TII->get(LMC.WQMOpc), LiveMaskWQM)992 .addReg(LiveMaskReg);993 NewTerm = BuildMI(MBB, MI, DL, TII->get(LMC.AndOpc), LMC.ExecReg)994 .addReg(LMC.ExecReg)995 .addReg(LiveMaskWQM);996 } else {997 // Kill - deactivate lanes no longer in live mask998 if (Op.isImm()) {999 NewTerm =1000 BuildMI(MBB, &MI, DL, TII->get(LMC.MovOpc), LMC.ExecReg).addImm(0);1001 } else if (!IsWQM) {1002 NewTerm = BuildMI(MBB, &MI, DL, TII->get(LMC.AndOpc), LMC.ExecReg)1003 .addReg(LMC.ExecReg)1004 .addReg(LiveMaskReg);1005 } else {1006 unsigned Opcode = KillVal ? LMC.AndN2Opc : LMC.AndOpc;1007 NewTerm = BuildMI(MBB, &MI, DL, TII->get(Opcode), LMC.ExecReg)1008 .addReg(LMC.ExecReg)1009 .add(Op);1010 }1011 }1012 1013 // Update live intervals1014 LIS->RemoveMachineInstrFromMaps(MI);1015 MBB.remove(&MI);1016 assert(EarlyTermMI);1017 assert(MaskUpdateMI);1018 assert(NewTerm);1019 if (ComputeKilledMaskMI)1020 LIS->InsertMachineInstrInMaps(*ComputeKilledMaskMI);1021 LIS->InsertMachineInstrInMaps(*MaskUpdateMI);1022 LIS->InsertMachineInstrInMaps(*EarlyTermMI);1023 if (WQMMaskMI)1024 LIS->InsertMachineInstrInMaps(*WQMMaskMI);1025 LIS->InsertMachineInstrInMaps(*NewTerm);1026 1027 if (CndReg) {1028 LIS->removeInterval(CndReg);1029 LIS->createAndComputeVirtRegInterval(CndReg);1030 }1031 if (TmpReg)1032 LIS->createAndComputeVirtRegInterval(TmpReg);1033 if (LiveMaskWQM)1034 LIS->createAndComputeVirtRegInterval(LiveMaskWQM);1035 1036 return NewTerm;1037}1038 1039// Replace (or supplement) instructions accessing live mask.1040// This can only happen once all the live mask registers have been created1041// and the execute state (WQM/StrictWWM/Exact) of instructions is known.1042void SIWholeQuadMode::lowerBlock(MachineBasicBlock &MBB, BlockInfo &BI) {1043 if (!BI.NeedsLowering)1044 return;1045 1046 LLVM_DEBUG(dbgs() << "\nLowering block " << printMBBReference(MBB) << ":\n");1047 1048 SmallVector<MachineInstr *, 4> SplitPoints;1049 Register ActiveLanesReg = 0;1050 char State = BI.InitialState;1051 1052 for (MachineInstr &MI : llvm::make_early_inc_range(1053 llvm::make_range(MBB.getFirstNonPHI(), MBB.end()))) {1054 auto MIState = StateTransition.find(&MI);1055 if (MIState != StateTransition.end())1056 State = MIState->second;1057 1058 MachineInstr *SplitPoint = nullptr;1059 switch (MI.getOpcode()) {1060 case AMDGPU::SI_DEMOTE_I1:1061 case AMDGPU::SI_KILL_I1_TERMINATOR:1062 SplitPoint = lowerKillI1(MI, State == StateWQM);1063 break;1064 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:1065 SplitPoint = lowerKillF32(MI);1066 break;1067 case AMDGPU::ENTER_STRICT_WWM:1068 ActiveLanesReg = MI.getOperand(0).getReg();1069 break;1070 case AMDGPU::EXIT_STRICT_WWM:1071 ActiveLanesReg = 0;1072 break;1073 case AMDGPU::V_SET_INACTIVE_B32:1074 if (ActiveLanesReg) {1075 LiveInterval &LI = LIS->getInterval(MI.getOperand(5).getReg());1076 MRI->constrainRegClass(ActiveLanesReg, TRI->getWaveMaskRegClass());1077 MI.getOperand(5).setReg(ActiveLanesReg);1078 LIS->shrinkToUses(&LI);1079 } else {1080 assert(State == StateExact || State == StateWQM);1081 }1082 break;1083 default:1084 break;1085 }1086 if (SplitPoint)1087 SplitPoints.push_back(SplitPoint);1088 }1089 1090 // Perform splitting after instruction scan to simplify iteration.1091 for (MachineInstr *MI : SplitPoints)1092 splitBlock(MI);1093}1094 1095// Return an iterator in the (inclusive) range [First, Last] at which1096// instructions can be safely inserted, keeping in mind that some of the1097// instructions we want to add necessarily clobber SCC.1098MachineBasicBlock::iterator SIWholeQuadMode::prepareInsertion(1099 MachineBasicBlock &MBB, MachineBasicBlock::iterator First,1100 MachineBasicBlock::iterator Last, bool PreferLast, bool SaveSCC) {1101 if (!SaveSCC)1102 return PreferLast ? Last : First;1103 1104 LiveRange &LR =1105 LIS->getRegUnit(*TRI->regunits(MCRegister::from(AMDGPU::SCC)).begin());1106 auto MBBE = MBB.end();1107 SlotIndex FirstIdx = First != MBBE ? LIS->getInstructionIndex(*First)1108 : LIS->getMBBEndIdx(&MBB);1109 SlotIndex LastIdx =1110 Last != MBBE ? LIS->getInstructionIndex(*Last) : LIS->getMBBEndIdx(&MBB);1111 SlotIndex Idx = PreferLast ? LastIdx : FirstIdx;1112 const LiveRange::Segment *S;1113 1114 for (;;) {1115 S = LR.getSegmentContaining(Idx);1116 if (!S)1117 break;1118 1119 if (PreferLast) {1120 SlotIndex Next = S->start.getBaseIndex();1121 if (Next < FirstIdx)1122 break;1123 Idx = Next;1124 } else {1125 MachineInstr *EndMI = LIS->getInstructionFromIndex(S->end.getBaseIndex());1126 assert(EndMI && "Segment does not end on valid instruction");1127 auto NextI = std::next(EndMI->getIterator());1128 if (NextI == MBB.end())1129 break;1130 SlotIndex Next = LIS->getInstructionIndex(*NextI);1131 if (Next > LastIdx)1132 break;1133 Idx = Next;1134 }1135 }1136 1137 MachineBasicBlock::iterator MBBI;1138 1139 if (MachineInstr *MI = LIS->getInstructionFromIndex(Idx))1140 MBBI = MI;1141 else {1142 assert(Idx == LIS->getMBBEndIdx(&MBB));1143 MBBI = MBB.end();1144 }1145 1146 // Move insertion point past any operations modifying EXEC.1147 // This assumes that the value of SCC defined by any of these operations1148 // does not need to be preserved.1149 while (MBBI != Last) {1150 bool IsExecDef = false;1151 for (const MachineOperand &MO : MBBI->all_defs()) {1152 IsExecDef |=1153 MO.getReg() == AMDGPU::EXEC_LO || MO.getReg() == AMDGPU::EXEC;1154 }1155 if (!IsExecDef)1156 break;1157 MBBI++;1158 S = nullptr;1159 }1160 1161 if (S)1162 MBBI = saveSCC(MBB, MBBI);1163 1164 return MBBI;1165}1166 1167void SIWholeQuadMode::toExact(MachineBasicBlock &MBB,1168 MachineBasicBlock::iterator Before,1169 Register SaveWQM) {1170 assert(LiveMaskReg.isVirtual());1171 1172 bool IsTerminator = Before == MBB.end();1173 if (!IsTerminator) {1174 auto FirstTerm = MBB.getFirstTerminator();1175 if (FirstTerm != MBB.end()) {1176 SlotIndex FirstTermIdx = LIS->getInstructionIndex(*FirstTerm);1177 SlotIndex BeforeIdx = LIS->getInstructionIndex(*Before);1178 IsTerminator = BeforeIdx > FirstTermIdx;1179 }1180 }1181 1182 MachineInstr *MI;1183 1184 if (SaveWQM) {1185 unsigned Opcode =1186 IsTerminator ? LMC.AndSaveExecTermOpc : LMC.AndSaveExecOpc;1187 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(Opcode), SaveWQM)1188 .addReg(LiveMaskReg);1189 } else {1190 unsigned Opcode = IsTerminator ? LMC.AndTermOpc : LMC.AndOpc;1191 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(Opcode), LMC.ExecReg)1192 .addReg(LMC.ExecReg)1193 .addReg(LiveMaskReg);1194 }1195 1196 LIS->InsertMachineInstrInMaps(*MI);1197 StateTransition[MI] = StateExact;1198}1199 1200void SIWholeQuadMode::toWQM(MachineBasicBlock &MBB,1201 MachineBasicBlock::iterator Before,1202 Register SavedWQM) {1203 MachineInstr *MI;1204 1205 if (SavedWQM) {1206 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::COPY), LMC.ExecReg)1207 .addReg(SavedWQM);1208 } else {1209 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(LMC.WQMOpc), LMC.ExecReg)1210 .addReg(LMC.ExecReg);1211 }1212 1213 LIS->InsertMachineInstrInMaps(*MI);1214 StateTransition[MI] = StateWQM;1215}1216 1217void SIWholeQuadMode::toStrictMode(MachineBasicBlock &MBB,1218 MachineBasicBlock::iterator Before,1219 Register SaveOrig, char StrictStateNeeded) {1220 MachineInstr *MI;1221 assert(SaveOrig);1222 assert(StrictStateNeeded == StateStrictWWM ||1223 StrictStateNeeded == StateStrictWQM);1224 1225 if (StrictStateNeeded == StateStrictWWM) {1226 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_STRICT_WWM),1227 SaveOrig)1228 .addImm(-1);1229 } else {1230 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_STRICT_WQM),1231 SaveOrig)1232 .addImm(-1);1233 }1234 LIS->InsertMachineInstrInMaps(*MI);1235 StateTransition[MI] = StrictStateNeeded;1236}1237 1238void SIWholeQuadMode::fromStrictMode(MachineBasicBlock &MBB,1239 MachineBasicBlock::iterator Before,1240 Register SavedOrig, char NonStrictState,1241 char CurrentStrictState) {1242 MachineInstr *MI;1243 1244 assert(SavedOrig);1245 assert(CurrentStrictState == StateStrictWWM ||1246 CurrentStrictState == StateStrictWQM);1247 1248 if (CurrentStrictState == StateStrictWWM) {1249 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_STRICT_WWM),1250 LMC.ExecReg)1251 .addReg(SavedOrig);1252 } else {1253 MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_STRICT_WQM),1254 LMC.ExecReg)1255 .addReg(SavedOrig);1256 }1257 LIS->InsertMachineInstrInMaps(*MI);1258 StateTransition[MI] = NonStrictState;1259}1260 1261void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, BlockInfo &BI,1262 bool IsEntry) {1263 // This is a non-entry block that is WQM throughout, so no need to do1264 // anything.1265 if (!IsEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact) {1266 BI.InitialState = StateWQM;1267 return;1268 }1269 1270 LLVM_DEBUG(dbgs() << "\nProcessing block " << printMBBReference(MBB)1271 << ":\n");1272 1273 Register SavedWQMReg;1274 Register SavedNonStrictReg;1275 bool WQMFromExec = IsEntry;1276 char State = (IsEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM;1277 char NonStrictState = 0;1278 const TargetRegisterClass *BoolRC = TRI->getBoolRC();1279 1280 auto II = MBB.getFirstNonPHI(), IE = MBB.end();1281 if (IsEntry) {1282 // Skip the instruction that saves LiveMask1283 if (II != IE && II->getOpcode() == AMDGPU::COPY &&1284 II->getOperand(1).getReg() == LMC.ExecReg)1285 ++II;1286 }1287 1288 // This stores the first instruction where it's safe to switch from WQM to1289 // Exact or vice versa.1290 MachineBasicBlock::iterator FirstWQM = IE;1291 1292 // This stores the first instruction where it's safe to switch from Strict1293 // mode to Exact/WQM or to switch to Strict mode. It must always be the same1294 // as, or after, FirstWQM since if it's safe to switch to/from Strict, it must1295 // be safe to switch to/from WQM as well.1296 MachineBasicBlock::iterator FirstStrict = IE;1297 1298 // Record initial state is block information.1299 BI.InitialState = State;1300 1301 for (unsigned Idx = 0;; ++Idx) {1302 MachineBasicBlock::iterator Next = II;1303 char Needs = StateExact | StateWQM; // Strict mode is disabled by default.1304 char OutNeeds = 0;1305 1306 if (FirstWQM == IE)1307 FirstWQM = II;1308 1309 if (FirstStrict == IE)1310 FirstStrict = II;1311 1312 // Adjust needs if this is first instruction of WQM requiring shader.1313 if (IsEntry && Idx == 0 && (BI.InNeeds & StateWQM))1314 Needs = StateWQM;1315 1316 // First, figure out the allowed states (Needs) based on the propagated1317 // flags.1318 if (II != IE) {1319 MachineInstr &MI = *II;1320 1321 if (MI.isTerminator() || TII->mayReadEXEC(*MRI, MI)) {1322 auto III = Instructions.find(&MI);1323 if (III != Instructions.end()) {1324 if (III->second.Needs & StateStrictWWM)1325 Needs = StateStrictWWM;1326 else if (III->second.Needs & StateStrictWQM)1327 Needs = StateStrictWQM;1328 else if (III->second.Needs & StateWQM)1329 Needs = StateWQM;1330 else1331 Needs &= ~III->second.Disabled;1332 OutNeeds = III->second.OutNeeds;1333 }1334 } else {1335 // If the instruction doesn't actually need a correct EXEC, then we can1336 // safely leave Strict mode enabled.1337 Needs = StateExact | StateWQM | StateStrict;1338 }1339 1340 // Exact mode exit can occur in terminators, but must be before branches.1341 if (MI.isBranch() && OutNeeds == StateExact)1342 Needs = StateExact;1343 1344 ++Next;1345 } else {1346 // End of basic block1347 if (BI.OutNeeds & StateWQM)1348 Needs = StateWQM;1349 else if (BI.OutNeeds == StateExact)1350 Needs = StateExact;1351 else1352 Needs = StateWQM | StateExact;1353 }1354 1355 // Now, transition if necessary.1356 if (!(Needs & State)) {1357 MachineBasicBlock::iterator First;1358 if (State == StateStrictWWM || Needs == StateStrictWWM ||1359 State == StateStrictWQM || Needs == StateStrictWQM) {1360 // We must switch to or from Strict mode.1361 First = FirstStrict;1362 } else {1363 // We only need to switch to/from WQM, so we can use FirstWQM.1364 First = FirstWQM;1365 }1366 1367 // Whether we need to save SCC depends on start and end states.1368 bool SaveSCC = false;1369 switch (State) {1370 case StateExact:1371 case StateStrictWWM:1372 case StateStrictWQM:1373 // Exact/Strict -> Strict: save SCC1374 // Exact/Strict -> WQM: save SCC if WQM mask is generated from exec1375 // Exact/Strict -> Exact: no save1376 SaveSCC = (Needs & StateStrict) || ((Needs & StateWQM) && WQMFromExec);1377 break;1378 case StateWQM:1379 // WQM -> Exact/Strict: save SCC1380 SaveSCC = !(Needs & StateWQM);1381 break;1382 default:1383 llvm_unreachable("Unknown state");1384 break;1385 }1386 char StartState = State & StateStrict ? NonStrictState : State;1387 bool WQMToExact =1388 StartState == StateWQM && (Needs & StateExact) && !(Needs & StateWQM);1389 bool ExactToWQM = StartState == StateExact && (Needs & StateWQM) &&1390 !(Needs & StateExact);1391 bool PreferLast = Needs == StateWQM;1392 // Exact regions in divergent control flow may run at EXEC=0, so try to1393 // exclude instructions with unexpected effects from them.1394 // FIXME: ideally we would branch over these when EXEC=0,1395 // but this requires updating implicit values, live intervals and CFG.1396 if ((WQMToExact && (OutNeeds & StateWQM)) || ExactToWQM) {1397 for (MachineBasicBlock::iterator I = First; I != II; ++I) {1398 if (TII->hasUnwantedEffectsWhenEXECEmpty(*I)) {1399 PreferLast = WQMToExact;1400 break;1401 }1402 }1403 }1404 MachineBasicBlock::iterator Before =1405 prepareInsertion(MBB, First, II, PreferLast, SaveSCC);1406 1407 if (State & StateStrict) {1408 assert(State == StateStrictWWM || State == StateStrictWQM);1409 assert(SavedNonStrictReg);1410 fromStrictMode(MBB, Before, SavedNonStrictReg, NonStrictState, State);1411 1412 LIS->createAndComputeVirtRegInterval(SavedNonStrictReg);1413 SavedNonStrictReg = 0;1414 State = NonStrictState;1415 }1416 1417 if (Needs & StateStrict) {1418 NonStrictState = State;1419 assert(Needs == StateStrictWWM || Needs == StateStrictWQM);1420 assert(!SavedNonStrictReg);1421 SavedNonStrictReg = MRI->createVirtualRegister(BoolRC);1422 1423 toStrictMode(MBB, Before, SavedNonStrictReg, Needs);1424 State = Needs;1425 } else {1426 if (WQMToExact) {1427 if (!WQMFromExec && (OutNeeds & StateWQM)) {1428 assert(!SavedWQMReg);1429 SavedWQMReg = MRI->createVirtualRegister(BoolRC);1430 }1431 1432 toExact(MBB, Before, SavedWQMReg);1433 State = StateExact;1434 } else if (ExactToWQM) {1435 assert(WQMFromExec == (SavedWQMReg == 0));1436 1437 toWQM(MBB, Before, SavedWQMReg);1438 1439 if (SavedWQMReg) {1440 LIS->createAndComputeVirtRegInterval(SavedWQMReg);1441 SavedWQMReg = 0;1442 }1443 State = StateWQM;1444 } else {1445 // We can get here if we transitioned from StrictWWM to a1446 // non-StrictWWM state that already matches our needs, but we1447 // shouldn't need to do anything.1448 assert(Needs & State);1449 }1450 }1451 }1452 1453 if (Needs != (StateExact | StateWQM | StateStrict)) {1454 if (Needs != (StateExact | StateWQM))1455 FirstWQM = IE;1456 FirstStrict = IE;1457 }1458 1459 if (II == IE)1460 break;1461 1462 II = Next;1463 }1464 assert(!SavedWQMReg);1465 assert(!SavedNonStrictReg);1466}1467 1468bool SIWholeQuadMode::lowerLiveMaskQueries() {1469 for (MachineInstr *MI : LiveMaskQueries) {1470 const DebugLoc &DL = MI->getDebugLoc();1471 Register Dest = MI->getOperand(0).getReg();1472 1473 MachineInstr *Copy =1474 BuildMI(*MI->getParent(), MI, DL, TII->get(AMDGPU::COPY), Dest)1475 .addReg(LiveMaskReg);1476 1477 LIS->ReplaceMachineInstrInMaps(*MI, *Copy);1478 MI->eraseFromParent();1479 }1480 return !LiveMaskQueries.empty();1481}1482 1483bool SIWholeQuadMode::lowerCopyInstrs() {1484 for (MachineInstr *MI : LowerToMovInstrs) {1485 assert(MI->getNumExplicitOperands() == 2);1486 1487 const Register Reg = MI->getOperand(0).getReg();1488 1489 const TargetRegisterClass *regClass =1490 TRI->getRegClassForOperandReg(*MRI, MI->getOperand(0));1491 if (TRI->isVGPRClass(regClass)) {1492 const unsigned MovOp = TII->getMovOpcode(regClass);1493 MI->setDesc(TII->get(MovOp));1494 1495 // Check that it already implicitly depends on exec (like all VALU movs1496 // should do).1497 assert(any_of(MI->implicit_operands(), [](const MachineOperand &MO) {1498 return MO.isUse() && MO.getReg() == AMDGPU::EXEC;1499 }));1500 } else {1501 // Remove early-clobber and exec dependency from simple SGPR copies.1502 // This allows some to be eliminated during/post RA.1503 LLVM_DEBUG(dbgs() << "simplify SGPR copy: " << *MI);1504 if (MI->getOperand(0).isEarlyClobber()) {1505 LIS->removeInterval(Reg);1506 MI->getOperand(0).setIsEarlyClobber(false);1507 LIS->createAndComputeVirtRegInterval(Reg);1508 }1509 int Index = MI->findRegisterUseOperandIdx(AMDGPU::EXEC, /*TRI=*/nullptr);1510 while (Index >= 0) {1511 MI->removeOperand(Index);1512 Index = MI->findRegisterUseOperandIdx(AMDGPU::EXEC, /*TRI=*/nullptr);1513 }1514 MI->setDesc(TII->get(AMDGPU::COPY));1515 LLVM_DEBUG(dbgs() << " -> " << *MI);1516 }1517 }1518 for (MachineInstr *MI : LowerToCopyInstrs) {1519 LLVM_DEBUG(dbgs() << "simplify: " << *MI);1520 1521 if (MI->getOpcode() == AMDGPU::V_SET_INACTIVE_B32) {1522 assert(MI->getNumExplicitOperands() == 6);1523 1524 LiveInterval *RecomputeLI = nullptr;1525 if (MI->getOperand(4).isReg())1526 RecomputeLI = &LIS->getInterval(MI->getOperand(4).getReg());1527 1528 MI->removeOperand(5);1529 MI->removeOperand(4);1530 MI->removeOperand(3);1531 MI->removeOperand(1);1532 1533 if (RecomputeLI)1534 LIS->shrinkToUses(RecomputeLI);1535 } else {1536 assert(MI->getNumExplicitOperands() == 2);1537 }1538 1539 unsigned CopyOp = MI->getOperand(1).isReg()1540 ? (unsigned)AMDGPU::COPY1541 : TII->getMovOpcode(TRI->getRegClassForOperandReg(1542 *MRI, MI->getOperand(0)));1543 MI->setDesc(TII->get(CopyOp));1544 LLVM_DEBUG(dbgs() << " -> " << *MI);1545 }1546 return !LowerToCopyInstrs.empty() || !LowerToMovInstrs.empty();1547}1548 1549bool SIWholeQuadMode::lowerKillInstrs(bool IsWQM) {1550 for (MachineInstr *MI : KillInstrs) {1551 MachineInstr *SplitPoint = nullptr;1552 switch (MI->getOpcode()) {1553 case AMDGPU::SI_DEMOTE_I1:1554 case AMDGPU::SI_KILL_I1_TERMINATOR:1555 SplitPoint = lowerKillI1(*MI, IsWQM);1556 break;1557 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:1558 SplitPoint = lowerKillF32(*MI);1559 break;1560 }1561 if (SplitPoint)1562 splitBlock(SplitPoint);1563 }1564 return !KillInstrs.empty();1565}1566 1567void SIWholeQuadMode::lowerInitExec(MachineInstr &MI) {1568 MachineBasicBlock *MBB = MI.getParent();1569 1570 if (MI.getOpcode() == AMDGPU::SI_INIT_WHOLE_WAVE) {1571 assert(MBB == &MBB->getParent()->front() &&1572 "init whole wave not in entry block");1573 Register EntryExec = MRI->createVirtualRegister(TRI->getBoolRC());1574 MachineInstr *SaveExec = BuildMI(*MBB, MBB->begin(), MI.getDebugLoc(),1575 TII->get(LMC.OrSaveExecOpc), EntryExec)1576 .addImm(-1);1577 1578 // Replace all uses of MI's destination reg with EntryExec.1579 MRI->replaceRegWith(MI.getOperand(0).getReg(), EntryExec);1580 1581 if (LIS) {1582 LIS->RemoveMachineInstrFromMaps(MI);1583 }1584 1585 MI.eraseFromParent();1586 1587 if (LIS) {1588 LIS->InsertMachineInstrInMaps(*SaveExec);1589 LIS->createAndComputeVirtRegInterval(EntryExec);1590 }1591 return;1592 }1593 1594 if (MI.getOpcode() == AMDGPU::SI_INIT_EXEC) {1595 // This should be before all vector instructions.1596 MachineInstr *InitMI = BuildMI(*MBB, MBB->begin(), MI.getDebugLoc(),1597 TII->get(LMC.MovOpc), LMC.ExecReg)1598 .addImm(MI.getOperand(0).getImm());1599 if (LIS) {1600 LIS->RemoveMachineInstrFromMaps(MI);1601 LIS->InsertMachineInstrInMaps(*InitMI);1602 }1603 MI.eraseFromParent();1604 return;1605 }1606 1607 // Extract the thread count from an SGPR input and set EXEC accordingly.1608 // Since BFM can't shift by 64, handle that case with CMP + CMOV.1609 //1610 // S_BFE_U32 count, input, {shift, 7}1611 // S_BFM_B64 exec, count, 01612 // S_CMP_EQ_U32 count, 641613 // S_CMOV_B64 exec, -11614 Register InputReg = MI.getOperand(0).getReg();1615 MachineInstr *FirstMI = &*MBB->begin();1616 if (InputReg.isVirtual()) {1617 MachineInstr *DefInstr = MRI->getVRegDef(InputReg);1618 assert(DefInstr && DefInstr->isCopy());1619 if (DefInstr->getParent() == MBB) {1620 if (DefInstr != FirstMI) {1621 // If the `InputReg` is defined in current block, we also need to1622 // move that instruction to the beginning of the block.1623 DefInstr->removeFromParent();1624 MBB->insert(FirstMI, DefInstr);1625 if (LIS)1626 LIS->handleMove(*DefInstr);1627 } else {1628 // If first instruction is definition then move pointer after it.1629 FirstMI = &*std::next(FirstMI->getIterator());1630 }1631 }1632 }1633 1634 // Insert instruction sequence at block beginning (before vector operations).1635 const DebugLoc DL = MI.getDebugLoc();1636 const unsigned WavefrontSize = ST->getWavefrontSize();1637 const unsigned Mask = (WavefrontSize << 1) - 1;1638 Register CountReg = MRI->createVirtualRegister(&AMDGPU::SGPR_32RegClass);1639 auto BfeMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_BFE_U32), CountReg)1640 .addReg(InputReg)1641 .addImm((MI.getOperand(1).getImm() & Mask) | 0x70000);1642 auto BfmMI = BuildMI(*MBB, FirstMI, DL, TII->get(LMC.BfmOpc), LMC.ExecReg)1643 .addReg(CountReg)1644 .addImm(0);1645 auto CmpMI = BuildMI(*MBB, FirstMI, DL, TII->get(AMDGPU::S_CMP_EQ_U32))1646 .addReg(CountReg, RegState::Kill)1647 .addImm(WavefrontSize);1648 auto CmovMI =1649 BuildMI(*MBB, FirstMI, DL, TII->get(LMC.CMovOpc), LMC.ExecReg).addImm(-1);1650 1651 if (!LIS) {1652 MI.eraseFromParent();1653 return;1654 }1655 1656 LIS->RemoveMachineInstrFromMaps(MI);1657 MI.eraseFromParent();1658 1659 LIS->InsertMachineInstrInMaps(*BfeMI);1660 LIS->InsertMachineInstrInMaps(*BfmMI);1661 LIS->InsertMachineInstrInMaps(*CmpMI);1662 LIS->InsertMachineInstrInMaps(*CmovMI);1663 1664 LIS->removeInterval(InputReg);1665 LIS->createAndComputeVirtRegInterval(InputReg);1666 LIS->createAndComputeVirtRegInterval(CountReg);1667}1668 1669/// Lower INIT_EXEC instructions. Return a suitable insert point in \p Entry1670/// for instructions that depend on EXEC.1671MachineBasicBlock::iterator1672SIWholeQuadMode::lowerInitExecInstrs(MachineBasicBlock &Entry, bool &Changed) {1673 MachineBasicBlock::iterator InsertPt = Entry.getFirstNonPHI();1674 1675 for (MachineInstr *MI : InitExecInstrs) {1676 // Try to handle undefined cases gracefully:1677 // - multiple INIT_EXEC instructions1678 // - INIT_EXEC instructions not in the entry block1679 if (MI->getParent() == &Entry)1680 InsertPt = std::next(MI->getIterator());1681 1682 lowerInitExec(*MI);1683 Changed = true;1684 }1685 1686 return InsertPt;1687}1688 1689bool SIWholeQuadMode::run(MachineFunction &MF) {1690 LLVM_DEBUG(dbgs() << "SI Whole Quad Mode on " << MF.getName()1691 << " ------------- \n");1692 LLVM_DEBUG(MF.dump(););1693 1694 Instructions.clear();1695 Blocks.clear();1696 LiveMaskQueries.clear();1697 LowerToCopyInstrs.clear();1698 LowerToMovInstrs.clear();1699 KillInstrs.clear();1700 InitExecInstrs.clear();1701 SetInactiveInstrs.clear();1702 StateTransition.clear();1703 1704 const char GlobalFlags = analyzeFunction(MF);1705 bool Changed = false;1706 1707 LiveMaskReg = LMC.ExecReg;1708 1709 MachineBasicBlock &Entry = MF.front();1710 MachineBasicBlock::iterator EntryMI = lowerInitExecInstrs(Entry, Changed);1711 1712 // Store a copy of the original live mask when required1713 const bool HasLiveMaskQueries = !LiveMaskQueries.empty();1714 const bool HasWaveModes = GlobalFlags & ~StateExact;1715 const bool HasKills = !KillInstrs.empty();1716 const bool UsesWQM = GlobalFlags & StateWQM;1717 if (HasKills || UsesWQM || (HasWaveModes && HasLiveMaskQueries)) {1718 LiveMaskReg = MRI->createVirtualRegister(TRI->getBoolRC());1719 MachineInstr *MI =1720 BuildMI(Entry, EntryMI, DebugLoc(), TII->get(AMDGPU::COPY), LiveMaskReg)1721 .addReg(LMC.ExecReg);1722 LIS->InsertMachineInstrInMaps(*MI);1723 Changed = true;1724 }1725 1726 // Check if V_SET_INACTIVE was touched by a strict state mode.1727 // If so, promote to WWM; otherwise lower to COPY.1728 for (MachineInstr *MI : SetInactiveInstrs) {1729 if (LowerToCopyInstrs.contains(MI))1730 continue;1731 auto &Info = Instructions[MI];1732 if (Info.MarkedStates & StateStrict) {1733 Info.Needs |= StateStrictWWM;1734 Info.Disabled &= ~StateStrictWWM;1735 Blocks[MI->getParent()].Needs |= StateStrictWWM;1736 } else {1737 LLVM_DEBUG(dbgs() << "Has no WWM marking: " << *MI);1738 LowerToCopyInstrs.insert(MI);1739 }1740 }1741 1742 LLVM_DEBUG(printInfo());1743 1744 Changed |= lowerLiveMaskQueries();1745 Changed |= lowerCopyInstrs();1746 1747 if (!HasWaveModes) {1748 // No wave mode execution1749 Changed |= lowerKillInstrs(false);1750 } else if (GlobalFlags == StateWQM) {1751 // Shader only needs WQM1752 auto MI =1753 BuildMI(Entry, EntryMI, DebugLoc(), TII->get(LMC.WQMOpc), LMC.ExecReg)1754 .addReg(LMC.ExecReg);1755 LIS->InsertMachineInstrInMaps(*MI);1756 lowerKillInstrs(true);1757 Changed = true;1758 } else {1759 // Mark entry for WQM if required.1760 if (GlobalFlags & StateWQM)1761 Blocks[&Entry].InNeeds |= StateWQM;1762 // Wave mode switching requires full lowering pass.1763 for (auto &BII : Blocks)1764 processBlock(*BII.first, BII.second, BII.first == &Entry);1765 // Lowering blocks causes block splitting so perform as a second pass.1766 for (auto &BII : Blocks)1767 lowerBlock(*BII.first, BII.second);1768 Changed = true;1769 }1770 1771 // Compute live range for live mask1772 if (LiveMaskReg != LMC.ExecReg)1773 LIS->createAndComputeVirtRegInterval(LiveMaskReg);1774 1775 // Physical registers like SCC aren't tracked by default anyway, so just1776 // removing the ranges we computed is the simplest option for maintaining1777 // the analysis results.1778 LIS->removeAllRegUnitsForPhysReg(AMDGPU::SCC);1779 1780 // If we performed any kills then recompute EXEC1781 if (!KillInstrs.empty() || !InitExecInstrs.empty())1782 LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);1783 1784 return Changed;1785}1786 1787bool SIWholeQuadModeLegacy::runOnMachineFunction(MachineFunction &MF) {1788 LiveIntervals *LIS = &getAnalysis<LiveIntervalsWrapperPass>().getLIS();1789 auto *MDTWrapper = getAnalysisIfAvailable<MachineDominatorTreeWrapperPass>();1790 MachineDominatorTree *MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr;1791 auto *PDTWrapper =1792 getAnalysisIfAvailable<MachinePostDominatorTreeWrapperPass>();1793 MachinePostDominatorTree *PDT =1794 PDTWrapper ? &PDTWrapper->getPostDomTree() : nullptr;1795 SIWholeQuadMode Impl(MF, LIS, MDT, PDT);1796 return Impl.run(MF);1797}1798 1799PreservedAnalyses1800SIWholeQuadModePass::run(MachineFunction &MF,1801 MachineFunctionAnalysisManager &MFAM) {1802 MFPropsModifier _(*this, MF);1803 1804 LiveIntervals *LIS = &MFAM.getResult<LiveIntervalsAnalysis>(MF);1805 MachineDominatorTree *MDT =1806 MFAM.getCachedResult<MachineDominatorTreeAnalysis>(MF);1807 MachinePostDominatorTree *PDT =1808 MFAM.getCachedResult<MachinePostDominatorTreeAnalysis>(MF);1809 SIWholeQuadMode Impl(MF, LIS, MDT, PDT);1810 bool Changed = Impl.run(MF);1811 if (!Changed)1812 return PreservedAnalyses::all();1813 1814 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();1815 PA.preserve<SlotIndexesAnalysis>();1816 PA.preserve<LiveIntervalsAnalysis>();1817 PA.preserve<MachineDominatorTreeAnalysis>();1818 PA.preserve<MachinePostDominatorTreeAnalysis>();1819 return PA;1820}1821