378 lines · cpp
1//===- AMDGPULowerVGPREncoding.cpp - lower VGPRs above v255 ---------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// Lower VGPRs above first 256 on gfx1250.11///12/// The pass scans used VGPRs and inserts S_SET_VGPR_MSB instructions to switch13/// VGPR addressing mode. The mode change is effective until the next change.14/// This instruction provides high bits of a VGPR address for four of the15/// operands: vdst, src0, src1, and src2, or other 4 operands depending on the16/// instruction encoding. If bits are set they are added as MSB to the17/// corresponding operand VGPR number.18///19/// There is no need to replace actual register operands because encoding of the20/// high and low VGPRs is the same. I.e. v0 has the encoding 0x100, so does21/// v256. v1 has the encoding 0x101 and v257 has the same encoding. So high22/// VGPRs will survive until actual encoding and will result in a same actual23/// bit encoding.24///25/// As a result the pass only inserts S_SET_VGPR_MSB to provide an actual offset26/// to a VGPR address of the subseqent instructions. The InstPrinter will take27/// care of the printing a low VGPR instead of a high one. In prinicple this28/// shall be viable to print actual high VGPR numbers, but that would disagree29/// with a disasm printing and create a situation where asm text is not30/// deterministic.31///32/// This pass creates a convention where non-fall through basic blocks shall33/// start with all 4 MSBs zero. Otherwise a disassembly would not be readable.34/// An optimization here is possible but deemed not desirable because of the35/// readbility concerns.36///37/// Consequentially the ABI is set to expect all 4 MSBs to be zero on entry.38/// The pass must run very late in the pipeline to make sure no changes to VGPR39/// operands will be made after it.40//41//===----------------------------------------------------------------------===//42 43#include "AMDGPULowerVGPREncoding.h"44#include "AMDGPU.h"45#include "GCNSubtarget.h"46#include "MCTargetDesc/AMDGPUMCTargetDesc.h"47#include "SIInstrInfo.h"48#include "llvm/ADT/PackedVector.h"49 50using namespace llvm;51 52#define DEBUG_TYPE "amdgpu-lower-vgpr-encoding"53 54namespace {55 56class AMDGPULowerVGPREncoding {57 static constexpr unsigned OpNum = 4;58 static constexpr unsigned BitsPerField = 2;59 static constexpr unsigned NumFields = 4;60 static constexpr unsigned FieldMask = (1 << BitsPerField) - 1;61 static constexpr unsigned ModeWidth = NumFields * BitsPerField;62 static constexpr unsigned ModeMask = (1 << ModeWidth) - 1;63 using ModeType = PackedVector<unsigned, BitsPerField,64 std::bitset<BitsPerField * NumFields>>;65 66 class ModeTy : public ModeType {67 public:68 // bitset constructor will set all bits to zero69 ModeTy() : ModeType(0) {}70 71 operator int64_t() const { return raw_bits().to_ulong(); }72 73 static ModeTy fullMask() {74 ModeTy M;75 M.raw_bits().flip();76 return M;77 }78 };79 80public:81 bool run(MachineFunction &MF);82 83private:84 const SIInstrInfo *TII;85 const SIRegisterInfo *TRI;86 87 // Current basic block.88 MachineBasicBlock *MBB;89 90 /// Most recent s_set_* instruction.91 MachineInstr *MostRecentModeSet;92 93 /// Current mode bits.94 ModeTy CurrentMode;95 96 /// Current mask of mode bits that instructions since MostRecentModeSet care97 /// about.98 ModeTy CurrentMask;99 100 /// Number of current hard clause instructions.101 unsigned ClauseLen;102 103 /// Number of hard clause instructions remaining.104 unsigned ClauseRemaining;105 106 /// Clause group breaks.107 unsigned ClauseBreaks;108 109 /// Last hard clause instruction.110 MachineInstr *Clause;111 112 /// Insert mode change before \p I. \returns true if mode was changed.113 bool setMode(ModeTy NewMode, ModeTy Mask,114 MachineBasicBlock::instr_iterator I);115 116 /// Reset mode to default.117 void resetMode(MachineBasicBlock::instr_iterator I) {118 setMode(ModeTy(), ModeTy::fullMask(), I);119 }120 121 /// If \p MO references VGPRs, return the MSBs. Otherwise, return nullopt.122 std::optional<unsigned> getMSBs(const MachineOperand &MO) const;123 124 /// Handle single \p MI. \return true if changed.125 bool runOnMachineInstr(MachineInstr &MI);126 127 /// Compute the mode and mode mask for a single \p MI given \p Ops operands128 /// bit mapping. Optionally takes second array \p Ops2 for VOPD.129 /// If provided and an operand from \p Ops is not a VGPR, then \p Ops2130 /// is checked.131 void computeMode(ModeTy &NewMode, ModeTy &Mask, MachineInstr &MI,132 const AMDGPU::OpName Ops[OpNum],133 const AMDGPU::OpName *Ops2 = nullptr);134 135 /// Check if an instruction \p I is within a clause and returns a suitable136 /// iterator to insert mode change. It may also modify the S_CLAUSE137 /// instruction to extend it or drop the clause if it cannot be adjusted.138 MachineBasicBlock::instr_iterator139 handleClause(MachineBasicBlock::instr_iterator I);140};141 142bool AMDGPULowerVGPREncoding::setMode(ModeTy NewMode, ModeTy Mask,143 MachineBasicBlock::instr_iterator I) {144 assert((NewMode.raw_bits() & ~Mask.raw_bits()).none());145 146 auto Delta = NewMode.raw_bits() ^ CurrentMode.raw_bits();147 148 if ((Delta & Mask.raw_bits()).none()) {149 CurrentMask |= Mask;150 return false;151 }152 153 if (MostRecentModeSet && (Delta & CurrentMask.raw_bits()).none()) {154 CurrentMode |= NewMode;155 CurrentMask |= Mask;156 157 MachineOperand &Op = MostRecentModeSet->getOperand(0);158 159 // Carry old mode bits from the existing instruction.160 int64_t OldModeBits = Op.getImm() & (ModeMask << ModeWidth);161 162 Op.setImm(CurrentMode | OldModeBits);163 return true;164 }165 166 // Record previous mode into high 8 bits of the immediate.167 int64_t OldModeBits = CurrentMode << ModeWidth;168 169 I = handleClause(I);170 MostRecentModeSet = BuildMI(*MBB, I, {}, TII->get(AMDGPU::S_SET_VGPR_MSB))171 .addImm(NewMode | OldModeBits);172 173 CurrentMode = NewMode;174 CurrentMask = Mask;175 return true;176}177 178std::optional<unsigned>179AMDGPULowerVGPREncoding::getMSBs(const MachineOperand &MO) const {180 if (!MO.isReg())181 return std::nullopt;182 183 MCRegister Reg = MO.getReg();184 const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);185 if (!RC || !TRI->isVGPRClass(RC))186 return std::nullopt;187 188 unsigned Idx = TRI->getHWRegIndex(Reg);189 return Idx >> 8;190}191 192void AMDGPULowerVGPREncoding::computeMode(ModeTy &NewMode, ModeTy &Mask,193 MachineInstr &MI,194 const AMDGPU::OpName Ops[OpNum],195 const AMDGPU::OpName *Ops2) {196 NewMode = {};197 Mask = {};198 199 for (unsigned I = 0; I < OpNum; ++I) {200 MachineOperand *Op = TII->getNamedOperand(MI, Ops[I]);201 202 std::optional<unsigned> MSBits;203 if (Op)204 MSBits = getMSBs(*Op);205 206#if !defined(NDEBUG)207 if (MSBits.has_value() && Ops2) {208 auto Op2 = TII->getNamedOperand(MI, Ops2[I]);209 if (Op2) {210 std::optional<unsigned> MSBits2;211 MSBits2 = getMSBs(*Op2);212 if (MSBits2.has_value() && MSBits != MSBits2)213 llvm_unreachable("Invalid VOPD pair was created");214 }215 }216#endif217 218 if (!MSBits.has_value() && Ops2) {219 Op = TII->getNamedOperand(MI, Ops2[I]);220 if (Op)221 MSBits = getMSBs(*Op);222 }223 224 if (!MSBits.has_value())225 continue;226 227 // Skip tied uses of src2 of VOP2, these will be handled along with defs and228 // only vdst bit affects these operands. We cannot skip tied uses of VOP3,229 // these uses are real even if must match the vdst.230 if (Ops[I] == AMDGPU::OpName::src2 && !Op->isDef() && Op->isTied() &&231 (SIInstrInfo::isVOP2(MI) ||232 (SIInstrInfo::isVOP3(MI) &&233 TII->hasVALU32BitEncoding(MI.getOpcode()))))234 continue;235 236 NewMode[I] = MSBits.value();237 Mask[I] = FieldMask;238 }239}240 241bool AMDGPULowerVGPREncoding::runOnMachineInstr(MachineInstr &MI) {242 auto Ops = AMDGPU::getVGPRLoweringOperandTables(MI.getDesc());243 if (Ops.first) {244 ModeTy NewMode, Mask;245 computeMode(NewMode, Mask, MI, Ops.first, Ops.second);246 return setMode(NewMode, Mask, MI.getIterator());247 }248 assert(!TII->hasVGPRUses(MI) || MI.isMetaInstruction() || MI.isPseudo());249 250 return false;251}252 253MachineBasicBlock::instr_iterator254AMDGPULowerVGPREncoding::handleClause(MachineBasicBlock::instr_iterator I) {255 if (!ClauseRemaining)256 return I;257 258 // A clause cannot start with a special instruction, place it right before259 // the clause.260 if (ClauseRemaining == ClauseLen) {261 I = Clause->getPrevNode()->getIterator();262 assert(I->isBundle());263 return I;264 }265 266 // If a clause defines breaks each group cannot start with a mode change.267 // just drop the clause.268 if (ClauseBreaks) {269 Clause->eraseFromBundle();270 ClauseRemaining = 0;271 return I;272 }273 274 // Otherwise adjust a number of instructions in the clause if it fits.275 // If it does not clause will just become shorter. Since the length276 // recorded in the clause is one less, increment the length after the277 // update. Note that SIMM16[5:0] must be 1-62, not 0 or 63.278 if (ClauseLen < 63)279 Clause->getOperand(0).setImm(ClauseLen | (ClauseBreaks << 8));280 281 ++ClauseLen;282 283 return I;284}285 286bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {287 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();288 if (!ST.has1024AddressableVGPRs())289 return false;290 291 TII = ST.getInstrInfo();292 TRI = ST.getRegisterInfo();293 294 bool Changed = false;295 ClauseLen = ClauseRemaining = 0;296 CurrentMode.reset();297 CurrentMask.reset();298 for (auto &MBB : MF) {299 MostRecentModeSet = nullptr;300 this->MBB = &MBB;301 302 for (auto &MI : llvm::make_early_inc_range(MBB.instrs())) {303 if (MI.isMetaInstruction())304 continue;305 306 if (MI.isTerminator() || MI.isCall()) {307 if (MI.getOpcode() == AMDGPU::S_ENDPGM ||308 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED)309 CurrentMode.reset();310 else311 resetMode(MI.getIterator());312 continue;313 }314 315 if (MI.isInlineAsm()) {316 if (TII->hasVGPRUses(MI))317 resetMode(MI.getIterator());318 continue;319 }320 321 if (MI.getOpcode() == AMDGPU::S_CLAUSE) {322 assert(!ClauseRemaining && "Nested clauses are not supported");323 ClauseLen = MI.getOperand(0).getImm();324 ClauseBreaks = (ClauseLen >> 8) & 15;325 ClauseLen = ClauseRemaining = (ClauseLen & 63) + 1;326 Clause = &MI;327 continue;328 }329 330 Changed |= runOnMachineInstr(MI);331 332 if (ClauseRemaining)333 --ClauseRemaining;334 }335 336 // Reset the mode if we are falling through.337 resetMode(MBB.instr_end());338 }339 340 return Changed;341}342 343class AMDGPULowerVGPREncodingLegacy : public MachineFunctionPass {344public:345 static char ID;346 347 AMDGPULowerVGPREncodingLegacy() : MachineFunctionPass(ID) {}348 349 bool runOnMachineFunction(MachineFunction &MF) override {350 return AMDGPULowerVGPREncoding().run(MF);351 }352 353 void getAnalysisUsage(AnalysisUsage &AU) const override {354 AU.setPreservesCFG();355 MachineFunctionPass::getAnalysisUsage(AU);356 }357};358 359} // namespace360 361char AMDGPULowerVGPREncodingLegacy::ID = 0;362 363char &llvm::AMDGPULowerVGPREncodingLegacyID = AMDGPULowerVGPREncodingLegacy::ID;364 365INITIALIZE_PASS(AMDGPULowerVGPREncodingLegacy, DEBUG_TYPE,366 "AMDGPU Lower VGPR Encoding", false, false)367 368PreservedAnalyses369AMDGPULowerVGPREncodingPass::run(MachineFunction &MF,370 MachineFunctionAnalysisManager &MFAM) {371 if (!AMDGPULowerVGPREncoding().run(MF))372 return PreservedAnalyses::all();373 374 PreservedAnalyses PA;375 PA.preserveSet<CFGAnalyses>();376 return PA;377}378