477 lines · cpp
1//===- AMDGPUMCInstLower.cpp - Lower AMDGPU MachineInstr to an MCInst -----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// Code to lower AMDGPU MachineInstrs to their corresponding MCInst.11//12//===----------------------------------------------------------------------===//13//14 15#include "AMDGPUMCInstLower.h"16#include "AMDGPU.h"17#include "AMDGPUAsmPrinter.h"18#include "AMDGPUMachineFunction.h"19#include "MCTargetDesc/AMDGPUInstPrinter.h"20#include "MCTargetDesc/AMDGPUMCExpr.h"21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"22#include "SIMachineFunctionInfo.h"23#include "llvm/CodeGen/MachineBasicBlock.h"24#include "llvm/CodeGen/MachineInstr.h"25#include "llvm/IR/Constants.h"26#include "llvm/IR/Function.h"27#include "llvm/IR/GlobalVariable.h"28#include "llvm/MC/MCCodeEmitter.h"29#include "llvm/MC/MCContext.h"30#include "llvm/MC/MCExpr.h"31#include "llvm/MC/MCInst.h"32#include "llvm/MC/MCObjectStreamer.h"33#include "llvm/MC/MCStreamer.h"34#include "llvm/Support/Endian.h"35#include "llvm/Support/ErrorHandling.h"36#include "llvm/Support/Format.h"37#include <algorithm>38 39using namespace llvm;40 41#include "AMDGPUGenMCPseudoLowering.inc"42 43AMDGPUMCInstLower::AMDGPUMCInstLower(MCContext &ctx,44 const TargetSubtargetInfo &st,45 const AsmPrinter &ap):46 Ctx(ctx), ST(st), AP(ap) { }47 48static AMDGPUMCExpr::Specifier getSpecifier(unsigned MOFlags) {49 switch (MOFlags) {50 default:51 return AMDGPUMCExpr::S_None;52 case SIInstrInfo::MO_GOTPCREL:53 case SIInstrInfo::MO_GOTPCREL64:54 return AMDGPUMCExpr::S_GOTPCREL;55 case SIInstrInfo::MO_GOTPCREL32_LO:56 return AMDGPUMCExpr::S_GOTPCREL32_LO;57 case SIInstrInfo::MO_GOTPCREL32_HI:58 return AMDGPUMCExpr::S_GOTPCREL32_HI;59 case SIInstrInfo::MO_REL32_LO:60 return AMDGPUMCExpr::S_REL32_LO;61 case SIInstrInfo::MO_REL32_HI:62 return AMDGPUMCExpr::S_REL32_HI;63 case SIInstrInfo::MO_REL64:64 return AMDGPUMCExpr::S_REL64;65 case SIInstrInfo::MO_ABS32_LO:66 return AMDGPUMCExpr::S_ABS32_LO;67 case SIInstrInfo::MO_ABS32_HI:68 return AMDGPUMCExpr::S_ABS32_HI;69 case SIInstrInfo::MO_ABS64:70 return AMDGPUMCExpr::S_ABS64;71 }72}73 74bool AMDGPUMCInstLower::lowerOperand(const MachineOperand &MO,75 MCOperand &MCOp) const {76 switch (MO.getType()) {77 default:78 break;79 case MachineOperand::MO_Immediate:80 MCOp = MCOperand::createImm(MO.getImm());81 return true;82 case MachineOperand::MO_Register:83 MCOp = MCOperand::createReg(AMDGPU::getMCReg(MO.getReg(), ST));84 return true;85 case MachineOperand::MO_MachineBasicBlock:86 MCOp = MCOperand::createExpr(87 MCSymbolRefExpr::create(MO.getMBB()->getSymbol(), Ctx));88 return true;89 case MachineOperand::MO_GlobalAddress: {90 const GlobalValue *GV = MO.getGlobal();91 SmallString<128> SymbolName;92 AP.getNameWithPrefix(SymbolName, GV);93 MCSymbol *Sym = Ctx.getOrCreateSymbol(SymbolName);94 const MCExpr *Expr =95 MCSymbolRefExpr::create(Sym, getSpecifier(MO.getTargetFlags()), Ctx);96 int64_t Offset = MO.getOffset();97 if (Offset != 0) {98 Expr = MCBinaryExpr::createAdd(Expr,99 MCConstantExpr::create(Offset, Ctx), Ctx);100 }101 MCOp = MCOperand::createExpr(Expr);102 return true;103 }104 case MachineOperand::MO_ExternalSymbol: {105 MCSymbol *Sym = Ctx.getOrCreateSymbol(StringRef(MO.getSymbolName()));106 const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(Sym, Ctx);107 MCOp = MCOperand::createExpr(Expr);108 return true;109 }110 case MachineOperand::MO_RegisterMask:111 // Regmasks are like implicit defs.112 return false;113 case MachineOperand::MO_MCSymbol:114 if (MO.getTargetFlags() == SIInstrInfo::MO_FAR_BRANCH_OFFSET) {115 MCSymbol *Sym = MO.getMCSymbol();116 MCOp = MCOperand::createExpr(Sym->getVariableValue());117 return true;118 }119 break;120 }121 llvm_unreachable("unknown operand type");122}123 124// Lower true16 D16 Pseudo instruction to d16_lo/d16_hi MCInst based on125// Dst/Data's .l/.h selection126void AMDGPUMCInstLower::lowerT16D16Helper(const MachineInstr *MI,127 MCInst &OutMI) const {128 unsigned Opcode = MI->getOpcode();129 const auto *TII = static_cast<const SIInstrInfo*>(ST.getInstrInfo());130 const SIRegisterInfo &TRI = TII->getRegisterInfo();131 const auto *Info = AMDGPU::getT16D16Helper(Opcode);132 133 llvm::AMDGPU::OpName OpName;134 if (TII->isDS(Opcode)) {135 if (MI->mayLoad())136 OpName = llvm::AMDGPU::OpName::vdst;137 else if (MI->mayStore())138 OpName = llvm::AMDGPU::OpName::data0;139 else140 llvm_unreachable("LDS load or store expected");141 } else {142 OpName = AMDGPU::hasNamedOperand(Opcode, llvm::AMDGPU::OpName::vdata)143 ? llvm::AMDGPU::OpName::vdata144 : llvm::AMDGPU::OpName::vdst;145 }146 147 // select Dst/Data148 int VDstOrVDataIdx = AMDGPU::getNamedOperandIdx(Opcode, OpName);149 const MachineOperand &MIVDstOrVData = MI->getOperand(VDstOrVDataIdx);150 151 // select hi/lo MCInst152 bool IsHi = AMDGPU::isHi16Reg(MIVDstOrVData.getReg(), TRI);153 Opcode = IsHi ? Info->HiOp : Info->LoOp;154 155 int MCOpcode = TII->pseudoToMCOpcode(Opcode);156 assert(MCOpcode != -1 &&157 "Pseudo instruction doesn't have a target-specific version");158 OutMI.setOpcode(MCOpcode);159 160 // lower operands161 for (int I = 0, E = MI->getNumExplicitOperands(); I < E; I++) {162 const MachineOperand &MO = MI->getOperand(I);163 MCOperand MCOp;164 if (I == VDstOrVDataIdx)165 MCOp = MCOperand::createReg(TRI.get32BitRegister(MIVDstOrVData.getReg()));166 else167 lowerOperand(MO, MCOp);168 OutMI.addOperand(MCOp);169 }170 171 if (AMDGPU::hasNamedOperand(MCOpcode, AMDGPU::OpName::vdst_in)) {172 MCOperand MCOp;173 lowerOperand(MIVDstOrVData, MCOp);174 OutMI.addOperand(MCOp);175 }176}177 178void AMDGPUMCInstLower::lowerT16FmaMixFP16(const MachineInstr *MI,179 MCInst &OutMI) const {180 unsigned Opcode = MI->getOpcode();181 const auto *TII = static_cast<const SIInstrInfo *>(ST.getInstrInfo());182 const SIRegisterInfo &TRI = TII->getRegisterInfo();183 184 int VDstIdx = AMDGPU::getNamedOperandIdx(Opcode, llvm::AMDGPU::OpName::vdst);185 const MachineOperand &VDst = MI->getOperand(VDstIdx);186 bool IsHi = AMDGPU::isHi16Reg(VDst.getReg(), TRI);187 switch (Opcode) {188 case AMDGPU::V_FMA_MIX_F16_t16:189 Opcode = IsHi ? AMDGPU::V_FMA_MIXHI_F16 : AMDGPU::V_FMA_MIXLO_F16;190 break;191 case AMDGPU::V_FMA_MIX_BF16_t16:192 Opcode = IsHi ? AMDGPU::V_FMA_MIXHI_BF16 : AMDGPU::V_FMA_MIXLO_BF16;193 break;194 }195 int MCOpcode = TII->pseudoToMCOpcode(Opcode);196 assert(MCOpcode != -1 &&197 "Pseudo instruction doesn't have a target-specific version");198 OutMI.setOpcode(MCOpcode);199 200 // lower operands201 for (int I = 0, E = MI->getNumExplicitOperands(); I < E; I++) {202 const MachineOperand &MO = MI->getOperand(I);203 MCOperand MCOp;204 if (I == VDstIdx)205 MCOp = MCOperand::createReg(TRI.get32BitRegister(VDst.getReg()));206 else207 lowerOperand(MO, MCOp);208 OutMI.addOperand(MCOp);209 }210}211 212void AMDGPUMCInstLower::lower(const MachineInstr *MI, MCInst &OutMI) const {213 unsigned Opcode = MI->getOpcode();214 const auto *TII = static_cast<const SIInstrInfo *>(ST.getInstrInfo());215 216 // FIXME: Should be able to handle this with lowerPseudoInstExpansion. We217 // need to select it to the subtarget specific version, and there's no way to218 // do that with a single pseudo source operation.219 if (Opcode == AMDGPU::S_SETPC_B64_return)220 Opcode = AMDGPU::S_SETPC_B64;221 else if (Opcode == AMDGPU::SI_CALL) {222 // SI_CALL is just S_SWAPPC_B64 with an additional operand to track the223 // called function (which we need to remove here).224 OutMI.setOpcode(TII->pseudoToMCOpcode(AMDGPU::S_SWAPPC_B64));225 MCOperand Dest, Src;226 lowerOperand(MI->getOperand(0), Dest);227 lowerOperand(MI->getOperand(1), Src);228 OutMI.addOperand(Dest);229 OutMI.addOperand(Src);230 return;231 } else if (Opcode == AMDGPU::SI_TCRETURN ||232 Opcode == AMDGPU::SI_TCRETURN_GFX) {233 // TODO: How to use branch immediate and avoid register+add?234 Opcode = AMDGPU::S_SETPC_B64;235 } else if (AMDGPU::getT16D16Helper(Opcode)) {236 lowerT16D16Helper(MI, OutMI);237 return;238 } else if (Opcode == AMDGPU::V_FMA_MIX_F16_t16 ||239 Opcode == AMDGPU::V_FMA_MIX_BF16_t16) {240 lowerT16FmaMixFP16(MI, OutMI);241 return;242 }243 244 int MCOpcode = TII->pseudoToMCOpcode(Opcode);245 if (MCOpcode == -1) {246 LLVMContext &C = MI->getMF()->getFunction().getContext();247 C.emitError("AMDGPUMCInstLower::lower - Pseudo instruction doesn't have "248 "a target-specific version: " + Twine(MI->getOpcode()));249 }250 251 OutMI.setOpcode(MCOpcode);252 253 for (const MachineOperand &MO : MI->explicit_operands()) {254 MCOperand MCOp;255 lowerOperand(MO, MCOp);256 OutMI.addOperand(MCOp);257 }258 259 int FIIdx = AMDGPU::getNamedOperandIdx(MCOpcode, AMDGPU::OpName::fi);260 if (FIIdx >= (int)OutMI.getNumOperands())261 OutMI.addOperand(MCOperand::createImm(0));262}263 264bool AMDGPUAsmPrinter::lowerOperand(const MachineOperand &MO,265 MCOperand &MCOp) const {266 const GCNSubtarget &STI = MF->getSubtarget<GCNSubtarget>();267 AMDGPUMCInstLower MCInstLowering(OutContext, STI, *this);268 return MCInstLowering.lowerOperand(MO, MCOp);269}270 271const MCExpr *AMDGPUAsmPrinter::lowerConstant(const Constant *CV,272 const Constant *BaseCV,273 uint64_t Offset) {274 275 // Intercept LDS variables with known addresses276 if (const GlobalVariable *GV = dyn_cast<const GlobalVariable>(CV)) {277 if (std::optional<uint32_t> Address =278 AMDGPUMachineFunction::getLDSAbsoluteAddress(*GV)) {279 auto *IntTy = Type::getInt32Ty(CV->getContext());280 return AsmPrinter::lowerConstant(ConstantInt::get(IntTy, *Address),281 BaseCV, Offset);282 }283 }284 285 if (const MCExpr *E = lowerAddrSpaceCast(TM, CV, OutContext))286 return E;287 return AsmPrinter::lowerConstant(CV, BaseCV, Offset);288}289 290static void emitVGPRBlockComment(const MachineInstr *MI, const SIInstrInfo *TII,291 const TargetRegisterInfo *TRI,292 const SIMachineFunctionInfo *MFI,293 MCStreamer &OS) {294 // The instruction will only transfer a subset of the registers in the block,295 // based on the mask that is stored in m0. We could search for the instruction296 // that sets m0, but most of the time we'll already have the mask stored in297 // the machine function info. Try to use that. This assumes that we only use298 // block loads/stores for CSR spills.299 Register RegBlock =300 TII->getNamedOperand(*MI, MI->mayLoad() ? AMDGPU::OpName::vdst301 : AMDGPU::OpName::vdata)302 ->getReg();303 Register FirstRegInBlock = TRI->getSubReg(RegBlock, AMDGPU::sub0);304 uint32_t Mask = MFI->getMaskForVGPRBlockOps(RegBlock);305 306 if (!Mask)307 return; // Nothing to report308 309 SmallString<512> TransferredRegs;310 for (unsigned I = 0; I < sizeof(Mask) * 8; ++I) {311 if (Mask & (1 << I)) {312 (llvm::Twine(" ") + TRI->getRegAsmName(FirstRegInBlock + I))313 .toVector(TransferredRegs);314 }315 }316 317 OS.emitRawComment(" transferring at most " + TransferredRegs);318}319 320void AMDGPUAsmPrinter::emitInstruction(const MachineInstr *MI) {321 // FIXME: Enable feature predicate checks once all the test pass.322 // AMDGPU_MC::verifyInstructionPredicates(MI->getOpcode(),323 // getSubtargetInfo().getFeatureBits());324 325 if (MCInst OutInst; lowerPseudoInstExpansion(MI, OutInst)) {326 EmitToStreamer(*OutStreamer, OutInst);327 return;328 }329 330 const GCNSubtarget &STI = MF->getSubtarget<GCNSubtarget>();331 AMDGPUMCInstLower MCInstLowering(OutContext, STI, *this);332 333 StringRef Err;334 if (!STI.getInstrInfo()->verifyInstruction(*MI, Err)) {335 LLVMContext &C = MI->getMF()->getFunction().getContext();336 C.emitError("Illegal instruction detected: " + Err);337 MI->print(errs());338 }339 340 if (MI->isBundle()) {341 const MachineBasicBlock *MBB = MI->getParent();342 MachineBasicBlock::const_instr_iterator I = ++MI->getIterator();343 while (I != MBB->instr_end() && I->isInsideBundle()) {344 emitInstruction(&*I);345 ++I;346 }347 } else {348 // We don't want these pseudo instructions encoded. They are349 // placeholder terminator instructions and should only be printed as350 // comments.351 if (MI->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) {352 if (isVerbose())353 OutStreamer->emitRawComment(" return to shader part epilog");354 return;355 }356 357 if (MI->getOpcode() == AMDGPU::WAVE_BARRIER) {358 if (isVerbose())359 OutStreamer->emitRawComment(" wave barrier");360 return;361 }362 363 if (MI->getOpcode() == AMDGPU::SCHED_BARRIER) {364 if (isVerbose()) {365 std::string HexString;366 raw_string_ostream HexStream(HexString);367 HexStream << format_hex(MI->getOperand(0).getImm(), 10, true);368 OutStreamer->emitRawComment(" sched_barrier mask(" + HexString + ")");369 }370 return;371 }372 373 if (MI->getOpcode() == AMDGPU::SCHED_GROUP_BARRIER) {374 if (isVerbose()) {375 std::string HexString;376 raw_string_ostream HexStream(HexString);377 HexStream << format_hex(MI->getOperand(0).getImm(), 10, true);378 OutStreamer->emitRawComment(379 " sched_group_barrier mask(" + HexString + ") size(" +380 Twine(MI->getOperand(1).getImm()) + ") SyncID(" +381 Twine(MI->getOperand(2).getImm()) + ")");382 }383 return;384 }385 386 if (MI->getOpcode() == AMDGPU::IGLP_OPT) {387 if (isVerbose()) {388 std::string HexString;389 raw_string_ostream HexStream(HexString);390 HexStream << format_hex(MI->getOperand(0).getImm(), 10, true);391 OutStreamer->emitRawComment(" iglp_opt mask(" + HexString + ")");392 }393 return;394 }395 396 if (MI->getOpcode() == AMDGPU::SI_MASKED_UNREACHABLE) {397 if (isVerbose())398 OutStreamer->emitRawComment(" divergent unreachable");399 return;400 }401 402 if (MI->isMetaInstruction()) {403 if (isVerbose())404 OutStreamer->emitRawComment(" meta instruction");405 return;406 }407 408 if (isVerbose())409 if (STI.getInstrInfo()->isBlockLoadStore(MI->getOpcode()))410 emitVGPRBlockComment(MI, STI.getInstrInfo(), STI.getRegisterInfo(),411 MF->getInfo<SIMachineFunctionInfo>(),412 *OutStreamer);413 414 if (isVerbose() && MI->getOpcode() == AMDGPU::S_SET_VGPR_MSB) {415 unsigned V = MI->getOperand(0).getImm() & 0xff;416 OutStreamer->AddComment(417 " msbs: dst=" + Twine(V >> 6) + " src0=" + Twine(V & 3) +418 " src1=" + Twine((V >> 2) & 3) + " src2=" + Twine((V >> 4) & 3));419 }420 421 MCInst TmpInst;422 MCInstLowering.lower(MI, TmpInst);423 EmitToStreamer(*OutStreamer, TmpInst);424 425#ifdef EXPENSIVE_CHECKS426 // Check getInstSizeInBytes on explicitly specified CPUs (it cannot427 // work correctly for the generic CPU).428 //429 // The isPseudo check really shouldn't be here, but unfortunately there are430 // some negative lit tests that depend on being able to continue through431 // here even when pseudo instructions haven't been lowered.432 //433 // We also overestimate branch sizes with the offset bug.434 if (!MI->isPseudo() && STI.isCPUStringValid(STI.getCPU()) &&435 (!STI.hasOffset3fBug() || !MI->isBranch())) {436 SmallVector<MCFixup, 4> Fixups;437 SmallVector<char, 16> CodeBytes;438 439 std::unique_ptr<MCCodeEmitter> InstEmitter(createAMDGPUMCCodeEmitter(440 *STI.getInstrInfo(), OutContext));441 InstEmitter->encodeInstruction(TmpInst, CodeBytes, Fixups, STI);442 443 assert(CodeBytes.size() == STI.getInstrInfo()->getInstSizeInBytes(*MI));444 }445#endif446 447 if (DumpCodeInstEmitter) {448 // Disassemble instruction/operands to text449 DisasmLines.resize(DisasmLines.size() + 1);450 std::string &DisasmLine = DisasmLines.back();451 raw_string_ostream DisasmStream(DisasmLine);452 453 AMDGPUInstPrinter InstPrinter(*TM.getMCAsmInfo(), *STI.getInstrInfo(),454 *STI.getRegisterInfo());455 InstPrinter.printInst(&TmpInst, 0, StringRef(), STI, DisasmStream);456 457 // Disassemble instruction/operands to hex representation.458 SmallVector<MCFixup, 4> Fixups;459 SmallVector<char, 16> CodeBytes;460 461 DumpCodeInstEmitter->encodeInstruction(462 TmpInst, CodeBytes, Fixups, MF->getSubtarget<MCSubtargetInfo>());463 HexLines.resize(HexLines.size() + 1);464 std::string &HexLine = HexLines.back();465 raw_string_ostream HexStream(HexLine);466 467 for (size_t i = 0; i < CodeBytes.size(); i += 4) {468 unsigned int CodeDWord =469 support::endian::read32le(CodeBytes.data() + i);470 HexStream << format("%s%08X", (i > 0 ? " " : ""), CodeDWord);471 }472 473 DisasmLineMaxLen = std::max(DisasmLineMaxLen, DisasmLine.size());474 }475 }476}477