561 lines · cpp
1//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ExecutionEngine/JITLink/JITLink.h"10 11#include "llvm/ADT/StringExtras.h"12#include "llvm/BinaryFormat/Magic.h"13#include "llvm/ExecutionEngine/JITLink/COFF.h"14#include "llvm/ExecutionEngine/JITLink/ELF.h"15#include "llvm/ExecutionEngine/JITLink/MachO.h"16#include "llvm/ExecutionEngine/JITLink/XCOFF.h"17#include "llvm/ExecutionEngine/JITLink/aarch64.h"18#include "llvm/ExecutionEngine/JITLink/loongarch.h"19#include "llvm/ExecutionEngine/JITLink/systemz.h"20#include "llvm/ExecutionEngine/JITLink/x86.h"21#include "llvm/ExecutionEngine/JITLink/x86_64.h"22#include "llvm/Support/raw_ostream.h"23 24using namespace llvm;25using namespace llvm::object;26 27#define DEBUG_TYPE "jitlink"28 29namespace {30 31enum JITLinkErrorCode { GenericJITLinkError = 1 };32 33// FIXME: This class is only here to support the transition to llvm::Error. It34// will be removed once this transition is complete. Clients should prefer to35// deal with the Error value directly, rather than converting to error_code.36class JITLinkerErrorCategory : public std::error_category {37public:38 const char *name() const noexcept override { return "runtimedyld"; }39 40 std::string message(int Condition) const override {41 switch (static_cast<JITLinkErrorCode>(Condition)) {42 case GenericJITLinkError:43 return "Generic JITLink error";44 }45 llvm_unreachable("Unrecognized JITLinkErrorCode");46 }47};48 49} // namespace50 51namespace llvm {52namespace jitlink {53 54char JITLinkError::ID = 0;55 56void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }57 58std::error_code JITLinkError::convertToErrorCode() const {59 static JITLinkerErrorCategory TheJITLinkerErrorCategory;60 return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);61}62 63const char *getGenericEdgeKindName(Edge::Kind K) {64 switch (K) {65 case Edge::Invalid:66 return "INVALID RELOCATION";67 case Edge::KeepAlive:68 return "Keep-Alive";69 default:70 return "<Unrecognized edge kind>";71 }72}73 74const char *getLinkageName(Linkage L) {75 switch (L) {76 case Linkage::Strong:77 return "strong";78 case Linkage::Weak:79 return "weak";80 }81 llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");82}83 84const char *getScopeName(Scope S) {85 switch (S) {86 case Scope::Default:87 return "default";88 case Scope::Hidden:89 return "hidden";90 case Scope::SideEffectsOnly:91 return "side-effects-only";92 case Scope::Local:93 return "local";94 }95 llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");96}97 98bool isCStringBlock(Block &B) {99 if (B.getSize() == 0) // Empty blocks are not valid C-strings.100 return false;101 102 // Zero-fill blocks of size one are valid empty strings.103 if (B.isZeroFill())104 return B.getSize() == 1;105 106 for (size_t I = 0; I != B.getSize() - 1; ++I)107 if (B.getContent()[I] == '\0')108 return false;109 110 return B.getContent()[B.getSize() - 1] == '\0';111}112 113raw_ostream &operator<<(raw_ostream &OS, const Block &B) {114 return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())115 << ": "116 << "size = " << formatv("{0:x8}", B.getSize()) << ", "117 << (B.isZeroFill() ? "zero-fill" : "content")118 << ", align = " << B.getAlignment()119 << ", align-ofs = " << B.getAlignmentOffset()120 << ", section = " << B.getSection().getName();121}122 123raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {124 OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")125 << " + " << formatv("{0:x8}", Sym.getOffset())126 << "): size: " << formatv("{0:x8}", Sym.getSize())127 << ", linkage: " << formatv("{0:6}", getLinkageName(Sym.getLinkage()))128 << ", scope: " << formatv("{0:8}", getScopeName(Sym.getScope())) << ", "129 << (Sym.isLive() ? "live" : "dead") << " - "130 << (Sym.hasName() ? *Sym.getName() : "<anonymous symbol>");131 return OS;132}133 134void printEdge(raw_ostream &OS, const Block &B, const Edge &E,135 StringRef EdgeKindName) {136 OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()137 << " + " << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName138 << " -> ";139 140 auto &TargetSym = E.getTarget();141 if (TargetSym.hasName())142 OS << TargetSym.getName();143 else {144 auto &TargetBlock = TargetSym.getBlock();145 auto &TargetSec = TargetBlock.getSection();146 orc::ExecutorAddr SecAddress(~uint64_t(0));147 for (auto *B : TargetSec.blocks())148 if (B->getAddress() < SecAddress)149 SecAddress = B->getAddress();150 151 orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;152 OS << TargetSym.getAddress() << " (section " << TargetSec.getName();153 if (SecDelta)154 OS << " + " << formatv("{0:x}", SecDelta);155 OS << " / block " << TargetBlock.getAddress();156 if (TargetSym.getOffset())157 OS << " + " << formatv("{0:x}", TargetSym.getOffset());158 OS << ")";159 }160 161 if (E.getAddend() != 0)162 OS << " + " << E.getAddend();163}164 165Section::~Section() {166 for (auto *Sym : Symbols)167 Sym->~Symbol();168 for (auto *B : Blocks)169 B->~Block();170}171 172LinkGraph::~LinkGraph() {173 for (auto *Sym : AbsoluteSymbols) {174 Sym->~Symbol();175 }176 for (auto *Sym : external_symbols()) {177 Sym->~Symbol();178 }179 ExternalSymbols.clear();180}181 182std::vector<Block *> LinkGraph::splitBlockImpl(std::vector<Block *> Blocks,183 SplitBlockCache *Cache) {184 assert(!Blocks.empty() && "Blocks must at least contain the original block");185 186 // Fix up content of all blocks.187 ArrayRef<char> Content = Blocks.front()->getContent();188 for (size_t I = 0; I != Blocks.size() - 1; ++I) {189 Blocks[I]->setContent(190 Content.slice(Blocks[I]->getAddress() - Blocks[0]->getAddress(),191 Blocks[I + 1]->getAddress() - Blocks[I]->getAddress()));192 }193 Blocks.back()->setContent(194 Content.slice(Blocks.back()->getAddress() - Blocks[0]->getAddress()));195 bool IsMutable = Blocks[0]->ContentMutable;196 for (auto *B : Blocks)197 B->ContentMutable = IsMutable;198 199 // Transfer symbols.200 {201 SplitBlockCache LocalBlockSymbolsCache;202 if (!Cache)203 Cache = &LocalBlockSymbolsCache;204 205 // Build cache if required.206 if (*Cache == std::nullopt) {207 *Cache = SplitBlockCache::value_type();208 209 for (auto *Sym : Blocks[0]->getSection().symbols())210 if (&Sym->getBlock() == Blocks[0])211 (*Cache)->push_back(Sym);212 llvm::sort(**Cache, [](const Symbol *LHS, const Symbol *RHS) {213 return LHS->getAddress() > RHS->getAddress();214 });215 }216 217 auto TransferSymbol = [](Symbol &Sym, Block &B) {218 Sym.setOffset(Sym.getAddress() - B.getAddress());219 Sym.setBlock(B);220 if (Sym.getSize() > B.getSize())221 Sym.setSize(B.getSize() - Sym.getOffset());222 };223 224 // Transfer symbols to all blocks except the last one.225 for (size_t I = 0; I != Blocks.size() - 1; ++I) {226 if ((*Cache)->empty())227 break;228 while (!(*Cache)->empty() &&229 (*Cache)->back()->getAddress() < Blocks[I + 1]->getAddress()) {230 TransferSymbol(*(*Cache)->back(), *Blocks[I]);231 (*Cache)->pop_back();232 }233 }234 // Transfer symbols to the last block, checking that all are in-range.235 while (!(*Cache)->empty()) {236 auto &Sym = *(*Cache)->back();237 (*Cache)->pop_back();238 assert(Sym.getAddress() >= Blocks.back()->getAddress() &&239 "Symbol address preceeds block");240 assert(Sym.getAddress() <= Blocks.back()->getRange().End &&241 "Symbol address starts past end of block");242 TransferSymbol(Sym, *Blocks.back());243 }244 }245 246 // Transfer edges.247 auto &Edges = Blocks[0]->Edges;248 llvm::sort(Edges, [](const Edge &LHS, const Edge &RHS) {249 return LHS.getOffset() < RHS.getOffset();250 });251 252 for (size_t I = Blocks.size() - 1; I != 0; --I) {253 254 // If all edges have been transferred then bail out.255 if (Edges.empty())256 break;257 258 Edge::OffsetT Delta = Blocks[I]->getAddress() - Blocks[0]->getAddress();259 260 // If no edges to move for this block then move to the next one.261 if (Edges.back().getOffset() < Delta)262 continue;263 264 size_t EI = Edges.size() - 1;265 while (EI != 0 && Edges[EI - 1].getOffset() >= Delta)266 --EI;267 268 for (size_t J = EI; J != Edges.size(); ++J) {269 Blocks[I]->Edges.push_back(std::move(Edges[J]));270 Blocks[I]->Edges.back().setOffset(Blocks[I]->Edges.back().getOffset() -271 Delta);272 }273 274 while (Edges.size() > EI)275 Edges.pop_back();276 }277 278 return Blocks;279}280 281void LinkGraph::dump(raw_ostream &OS) {282 DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;283 284 OS << "LinkGraph \"" << getName()285 << "\" (triple = " << getTargetTriple().str() << ")\n";286 287 // Map from blocks to the symbols pointing at them.288 for (auto *Sym : defined_symbols())289 BlockSymbols[&Sym->getBlock()].push_back(Sym);290 291 // For each block, sort its symbols by something approximating292 // relevance.293 for (auto &KV : BlockSymbols)294 llvm::sort(KV.second, [](const Symbol *LHS, const Symbol *RHS) {295 if (LHS->getOffset() != RHS->getOffset())296 return LHS->getOffset() < RHS->getOffset();297 if (LHS->getLinkage() != RHS->getLinkage())298 return LHS->getLinkage() < RHS->getLinkage();299 if (LHS->getScope() != RHS->getScope())300 return LHS->getScope() < RHS->getScope();301 if (LHS->hasName()) {302 if (!RHS->hasName())303 return true;304 return LHS->getName() < RHS->getName();305 }306 return false;307 });308 309 std::vector<Section *> SortedSections;310 for (auto &Sec : sections())311 SortedSections.push_back(&Sec);312 llvm::sort(SortedSections, [](const Section *LHS, const Section *RHS) {313 return LHS->getName() < RHS->getName();314 });315 316 for (auto *Sec : SortedSections) {317 OS << "section " << Sec->getName() << ":\n\n";318 319 std::vector<Block *> SortedBlocks;320 llvm::append_range(SortedBlocks, Sec->blocks());321 llvm::sort(SortedBlocks, [](const Block *LHS, const Block *RHS) {322 return LHS->getAddress() < RHS->getAddress();323 });324 325 for (auto *B : SortedBlocks) {326 OS << " block " << B->getAddress()327 << " size = " << formatv("{0:x8}", B->getSize())328 << ", align = " << B->getAlignment()329 << ", alignment-offset = " << B->getAlignmentOffset();330 if (B->isZeroFill())331 OS << ", zero-fill";332 OS << "\n";333 334 auto BlockSymsI = BlockSymbols.find(B);335 if (BlockSymsI != BlockSymbols.end()) {336 OS << " symbols:\n";337 auto &Syms = BlockSymsI->second;338 for (auto *Sym : Syms)339 OS << " " << *Sym << "\n";340 } else341 OS << " no symbols\n";342 343 if (!B->edges_empty()) {344 OS << " edges:\n";345 std::vector<Edge> SortedEdges;346 llvm::append_range(SortedEdges, B->edges());347 llvm::sort(SortedEdges, [](const Edge &LHS, const Edge &RHS) {348 return LHS.getOffset() < RHS.getOffset();349 });350 for (auto &E : SortedEdges) {351 OS << " " << B->getFixupAddress(E) << " (block + "352 << formatv("{0:x8}", E.getOffset()) << "), addend = ";353 if (E.getAddend() >= 0)354 OS << formatv("+{0:x8}", E.getAddend());355 else356 OS << formatv("-{0:x8}", -E.getAddend());357 OS << ", kind = " << getEdgeKindName(E.getKind()) << ", target = ";358 if (E.getTarget().hasName())359 OS << E.getTarget().getName();360 else361 OS << "addressable@"362 << formatv("{0:x16}", E.getTarget().getAddress()) << "+"363 << formatv("{0:x8}", E.getTarget().getOffset());364 OS << "\n";365 }366 } else367 OS << " no edges\n";368 OS << "\n";369 }370 }371 372 OS << "Absolute symbols:\n";373 if (!absolute_symbols().empty()) {374 for (auto *Sym : absolute_symbols())375 OS << " " << Sym->getAddress() << ": " << *Sym << "\n";376 } else377 OS << " none\n";378 379 OS << "\nExternal symbols:\n";380 if (!external_symbols().empty()) {381 for (auto *Sym : external_symbols())382 OS << " " << Sym->getAddress() << ": " << *Sym383 << (Sym->isWeaklyReferenced() ? " (weakly referenced)" : "") << "\n";384 } else385 OS << " none\n";386}387 388raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {389 switch (LF) {390 case SymbolLookupFlags::RequiredSymbol:391 return OS << "RequiredSymbol";392 case SymbolLookupFlags::WeaklyReferencedSymbol:393 return OS << "WeaklyReferencedSymbol";394 }395 llvm_unreachable("Unrecognized lookup flags");396}397 398void JITLinkAsyncLookupContinuation::anchor() {}399 400JITLinkContext::~JITLinkContext() = default;401 402bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {403 return true;404}405 406LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {407 return LinkGraphPassFunction();408}409 410Error JITLinkContext::modifyPassConfig(LinkGraph &G,411 PassConfiguration &Config) {412 return Error::success();413}414 415Error markAllSymbolsLive(LinkGraph &G) {416 for (auto *Sym : G.defined_symbols())417 Sym->setLive(true);418 return Error::success();419}420 421Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,422 const Edge &E) {423 std::string ErrMsg;424 {425 raw_string_ostream ErrStream(ErrMsg);426 Section &Sec = B.getSection();427 ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()428 << ": relocation target "429 << formatv("{0:x}", E.getTarget().getAddress() + E.getAddend())430 << " (";431 if (E.getTarget().hasName())432 ErrStream << E.getTarget().getName();433 else434 ErrStream << "<anonymous symbol>";435 if (E.getAddend()) {436 // Target address includes non-zero added, so break down the arithmetic.437 ErrStream << formatv(":{0:x}", E.getTarget().getAddress()) << " + "438 << formatv("{0:x}", E.getAddend());439 }440 ErrStream << ") is out of range of " << G.getEdgeKindName(E.getKind())441 << " fixup at address "442 << formatv("{0:x}", E.getTarget().getAddress()) << " (";443 444 Symbol *BestSymbolForBlock = nullptr;445 for (auto *Sym : Sec.symbols())446 if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&447 (!BestSymbolForBlock ||448 Sym->getScope() < BestSymbolForBlock->getScope() ||449 Sym->getLinkage() < BestSymbolForBlock->getLinkage()))450 BestSymbolForBlock = Sym;451 452 if (BestSymbolForBlock)453 ErrStream << BestSymbolForBlock->getName() << ", ";454 else455 ErrStream << "<anonymous block> @ ";456 457 ErrStream << formatv("{0:x}", B.getAddress()) << " + "458 << formatv("{0:x}", E.getOffset()) << ")";459 }460 return make_error<JITLinkError>(std::move(ErrMsg));461}462 463Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,464 const Edge &E) {465 return make_error<JITLinkError>("0x" + llvm::utohexstr(Loc.getValue()) +466 " improper alignment for relocation " +467 formatv("{0:d}", E.getKind()) + ": 0x" +468 llvm::utohexstr(Value) +469 " is not aligned to " + Twine(N) + " bytes");470}471 472AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {473 switch (TT.getArch()) {474 case Triple::aarch64:475 return aarch64::createAnonymousPointer;476 case Triple::x86_64:477 return x86_64::createAnonymousPointer;478 case Triple::x86:479 return x86::createAnonymousPointer;480 case Triple::loongarch32:481 case Triple::loongarch64:482 return loongarch::createAnonymousPointer;483 case Triple::systemz:484 return systemz::createAnonymousPointer;485 default:486 return nullptr;487 }488}489 490PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {491 switch (TT.getArch()) {492 case Triple::aarch64:493 return aarch64::createAnonymousPointerJumpStub;494 case Triple::x86_64:495 return x86_64::createAnonymousPointerJumpStub;496 case Triple::x86:497 return x86::createAnonymousPointerJumpStub;498 case Triple::loongarch32:499 case Triple::loongarch64:500 return loongarch::createAnonymousPointerJumpStub;501 case Triple::systemz:502 return systemz::createAnonymousPointerJumpStub;503 default:504 return nullptr;505 }506}507 508Expected<std::unique_ptr<LinkGraph>>509createLinkGraphFromObject(MemoryBufferRef ObjectBuffer,510 std::shared_ptr<orc::SymbolStringPool> SSP) {511 auto Magic = identify_magic(ObjectBuffer.getBuffer());512 switch (Magic) {513 case file_magic::macho_object:514 return createLinkGraphFromMachOObject(ObjectBuffer, std::move(SSP));515 case file_magic::elf_relocatable:516 return createLinkGraphFromELFObject(ObjectBuffer, std::move(SSP));517 case file_magic::coff_object:518 return createLinkGraphFromCOFFObject(ObjectBuffer, std::move(SSP));519 case file_magic::xcoff_object_64:520 return createLinkGraphFromXCOFFObject(ObjectBuffer, std::move(SSP));521 default:522 return make_error<JITLinkError>("Unsupported file format");523 };524}525 526std::unique_ptr<LinkGraph>527absoluteSymbolsLinkGraph(Triple TT, std::shared_ptr<orc::SymbolStringPool> SSP,528 orc::SymbolMap Symbols) {529 static std::atomic<uint64_t> Counter = {0};530 auto Index = Counter.fetch_add(1, std::memory_order_relaxed);531 auto G = std::make_unique<LinkGraph>(532 "<Absolute Symbols " + std::to_string(Index) + ">", std::move(SSP),533 std::move(TT), SubtargetFeatures(), getGenericEdgeKindName);534 for (auto &[Name, Def] : Symbols) {535 auto &Sym =536 G->addAbsoluteSymbol(*Name, Def.getAddress(), /*Size=*/0,537 Linkage::Strong, Scope::Default, /*IsLive=*/true);538 Sym.setCallable(Def.getFlags().isCallable());539 }540 541 return G;542}543 544void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {545 switch (G->getTargetTriple().getObjectFormat()) {546 case Triple::MachO:547 return link_MachO(std::move(G), std::move(Ctx));548 case Triple::ELF:549 return link_ELF(std::move(G), std::move(Ctx));550 case Triple::COFF:551 return link_COFF(std::move(G), std::move(Ctx));552 case Triple::XCOFF:553 return link_XCOFF(std::move(G), std::move(Ctx));554 default:555 Ctx->notifyFailed(make_error<JITLinkError>("Unsupported object format"));556 };557}558 559} // end namespace jitlink560} // end namespace llvm561