397 lines · cpp
1//===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// ELF/x86-64 jit-link implementation.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"14#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"15#include "llvm/ExecutionEngine/JITLink/JITLink.h"16#include "llvm/ExecutionEngine/JITLink/TableManager.h"17#include "llvm/ExecutionEngine/JITLink/x86_64.h"18#include "llvm/Object/ELFObjectFile.h"19 20#include "DefineExternalSectionStartAndEndSymbols.h"21#include "EHFrameSupportImpl.h"22#include "ELFLinkGraphBuilder.h"23#include "JITLinkGeneric.h"24 25#define DEBUG_TYPE "jitlink"26 27using namespace llvm;28using namespace llvm::jitlink;29 30namespace {31 32constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";33constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";34 35class TLSInfoTableManager_ELF_x86_6436 : public TableManager<TLSInfoTableManager_ELF_x86_64> {37public:38 static const uint8_t TLSInfoEntryContent[16];39 40 static StringRef getSectionName() { return ELFTLSInfoSectionName; }41 42 bool visitEdge(LinkGraph &G, Block *B, Edge &E) {43 if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {44 LLVM_DEBUG({45 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "46 << formatv("{0:x}", B->getFixupAddress(E)) << " ("47 << formatv("{0:x}", B->getAddress()) << " + "48 << formatv("{0:x}", E.getOffset()) << ")\n";49 });50 E.setKind(x86_64::Delta32);51 E.setTarget(getEntryForTarget(G, E.getTarget()));52 return true;53 }54 return false;55 }56 57 Symbol &createEntry(LinkGraph &G, Symbol &Target) {58 // the TLS Info entry's key value will be written by the fixTLVSectionByName59 // pass, so create mutable content.60 auto &TLSInfoEntry = G.createMutableContentBlock(61 getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()),62 orc::ExecutorAddr(), 8, 0);63 TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);64 return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);65 }66 67private:68 Section &getTLSInfoSection(LinkGraph &G) {69 if (!TLSInfoTable)70 TLSInfoTable =71 &G.createSection(ELFTLSInfoSectionName, orc::MemProt::Read);72 return *TLSInfoTable;73 }74 75 ArrayRef<char> getTLSInfoEntryContent() const {76 return {reinterpret_cast<const char *>(TLSInfoEntryContent),77 sizeof(TLSInfoEntryContent)};78 }79 80 Section *TLSInfoTable = nullptr;81};82 83const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = {84 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */85 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/86};87 88Error buildTables_ELF_x86_64(LinkGraph &G) {89 LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");90 91 x86_64::GOTTableManager GOT(G);92 x86_64::PLTTableManager PLT(G, GOT);93 TLSInfoTableManager_ELF_x86_64 TLSInfo;94 visitExistingEdges(G, GOT, PLT, TLSInfo);95 return Error::success();96}97} // namespace98 99namespace llvm {100namespace jitlink {101 102class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {103private:104 using ELFT = object::ELF64LE;105 106 Error addRelocations() override {107 LLVM_DEBUG(dbgs() << "Processing relocations:\n");108 109 using Base = ELFLinkGraphBuilder<ELFT>;110 using Self = ELFLinkGraphBuilder_x86_64;111 for (const auto &RelSect : Base::Sections) {112 // Validate the section to read relocation entries from.113 if (RelSect.sh_type == ELF::SHT_REL)114 return make_error<StringError>(115 "No SHT_REL in valid x64 ELF object files",116 inconvertibleErrorCode());117 118 if (Error Err = Base::forEachRelaRelocation(RelSect, this,119 &Self::addSingleRelocation))120 return Err;121 }122 123 return Error::success();124 }125 126 Error addSingleRelocation(const typename ELFT::Rela &Rel,127 const typename ELFT::Shdr &FixupSection,128 Block &BlockToFix) {129 using Base = ELFLinkGraphBuilder<ELFT>;130 131 auto ELFReloc = Rel.getType(false);132 133 // R_X86_64_NONE is a no-op.134 if (LLVM_UNLIKELY(ELFReloc == ELF::R_X86_64_NONE))135 return Error::success();136 137 uint32_t SymbolIndex = Rel.getSymbol(false);138 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);139 if (!ObjSymbol)140 return ObjSymbol.takeError();141 142 Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);143 if (!GraphSymbol)144 return make_error<StringError>(145 formatv("Could not find symbol at given index, did you add it to "146 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",147 SymbolIndex, (*ObjSymbol)->st_shndx,148 Base::GraphSymbols.size()),149 inconvertibleErrorCode());150 151 // Validate the relocation kind.152 int64_t Addend = Rel.r_addend;153 Edge::Kind Kind = Edge::Invalid;154 155 switch (ELFReloc) {156 case ELF::R_X86_64_PC8:157 Kind = x86_64::Delta8;158 break;159 case ELF::R_X86_64_PC16:160 Kind = x86_64::Delta16;161 break;162 case ELF::R_X86_64_PC32:163 case ELF::R_X86_64_GOTPC32:164 Kind = x86_64::Delta32;165 break;166 case ELF::R_X86_64_PC64:167 case ELF::R_X86_64_GOTPC64:168 Kind = x86_64::Delta64;169 break;170 case ELF::R_X86_64_32:171 Kind = x86_64::Pointer32;172 break;173 case ELF::R_X86_64_16:174 Kind = x86_64::Pointer16;175 break;176 case ELF::R_X86_64_8:177 Kind = x86_64::Pointer8;178 break;179 case ELF::R_X86_64_32S:180 Kind = x86_64::Pointer32Signed;181 break;182 case ELF::R_X86_64_64:183 Kind = x86_64::Pointer64;184 break;185 case ELF::R_X86_64_SIZE32:186 Kind = x86_64::Size32;187 break;188 case ELF::R_X86_64_SIZE64:189 Kind = x86_64::Size64;190 break;191 case ELF::R_X86_64_GOTPCREL:192 Kind = x86_64::RequestGOTAndTransformToDelta32;193 break;194 case ELF::R_X86_64_REX_GOTPCRELX:195 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;196 Addend = 0;197 break;198 case ELF::R_X86_64_TLSGD:199 Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;200 break;201 case ELF::R_X86_64_GOTPCRELX:202 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;203 Addend = 0;204 break;205 case ELF::R_X86_64_GOTPCREL64:206 Kind = x86_64::RequestGOTAndTransformToDelta64;207 break;208 case ELF::R_X86_64_GOT64:209 Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;210 break;211 case ELF::R_X86_64_GOTOFF64:212 Kind = x86_64::Delta64FromGOT;213 break;214 case ELF::R_X86_64_PLT32:215 Kind = x86_64::BranchPCRel32;216 // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to217 // adjust the addend by '+4' to compensate.218 Addend += 4;219 break;220 default:221 return make_error<JITLinkError>(222 "In " + G->getName() + ": Unsupported x86-64 relocation type " +223 object::getELFRelocationTypeName(ELF::EM_X86_64, ELFReloc));224 }225 226 auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset;227 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();228 Edge GE(Kind, Offset, *GraphSymbol, Addend);229 LLVM_DEBUG({230 dbgs() << " ";231 printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind));232 dbgs() << "\n";233 });234 235 BlockToFix.addEdge(std::move(GE));236 return Error::success();237 }238 239public:240 ELFLinkGraphBuilder_x86_64(StringRef FileName,241 std::shared_ptr<orc::SymbolStringPool> SSP,242 const object::ELFFile<object::ELF64LE> &Obj,243 SubtargetFeatures Features)244 : ELFLinkGraphBuilder(Obj, std::move(SSP), Triple("x86_64-unknown-linux"),245 std::move(Features), FileName,246 x86_64::getEdgeKindName) {}247};248 249class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {250 friend class JITLinker<ELFJITLinker_x86_64>;251 252public:253 ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,254 std::unique_ptr<LinkGraph> G,255 PassConfiguration PassConfig)256 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {257 258 if (shouldAddDefaultTargetPasses(getGraph().getTargetTriple()))259 getPassConfig().PostAllocationPasses.push_back(260 [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });261 }262 263private:264 Symbol *GOTSymbol = nullptr;265 Error getOrCreateGOTSymbol(LinkGraph &G) {266 auto DefineExternalGOTSymbolIfPresent =267 createDefineExternalSectionStartAndEndSymbolsPass(268 [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {269 if (Sym.getName() != nullptr &&270 *Sym.getName() == ELFGOTSymbolName)271 if (auto *GOTSection = G.findSectionByName(272 x86_64::GOTTableManager::getSectionName())) {273 GOTSymbol = &Sym;274 return {*GOTSection, true};275 }276 return {};277 });278 279 // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an280 // external.281 if (auto Err = DefineExternalGOTSymbolIfPresent(G))282 return Err;283 284 // If we succeeded then we're done.285 if (GOTSymbol)286 return Error::success();287 288 // Otherwise look for a GOT section: If it already has a start symbol we'll289 // record it, otherwise we'll create our own.290 // If there's a GOT section but we didn't find an external GOT symbol...291 if (auto *GOTSection =292 G.findSectionByName(x86_64::GOTTableManager::getSectionName())) {293 294 // Check for an existing defined symbol.295 for (auto *Sym : GOTSection->symbols())296 if (Sym->getName() != nullptr && *Sym->getName() == ELFGOTSymbolName) {297 GOTSymbol = Sym;298 return Error::success();299 }300 301 // If there's no defined symbol then create one.302 SectionRange SR(*GOTSection);303 if (SR.empty())304 GOTSymbol =305 &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0,306 Linkage::Strong, Scope::Local, true);307 else308 GOTSymbol =309 &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,310 Linkage::Strong, Scope::Local, false, true);311 }312 313 // If we still haven't found a GOT symbol then double check the externals.314 // We may have a GOT-relative reference but no GOT section, in which case315 // we just need to point the GOT symbol at some address in this graph.316 if (!GOTSymbol) {317 for (auto *Sym : G.external_symbols()) {318 if (*Sym->getName() == ELFGOTSymbolName) {319 auto Blocks = G.blocks();320 if (!Blocks.empty()) {321 G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress());322 GOTSymbol = Sym;323 break;324 }325 }326 }327 }328 329 return Error::success();330 }331 332 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {333 return x86_64::applyFixup(G, B, E, GOTSymbol);334 }335};336 337Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_x86_64(338 MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {339 LLVM_DEBUG({340 dbgs() << "Building jitlink graph for new input "341 << ObjectBuffer.getBufferIdentifier() << "...\n";342 });343 344 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);345 if (!ELFObj)346 return ELFObj.takeError();347 348 auto Features = (*ELFObj)->getFeatures();349 if (!Features)350 return Features.takeError();351 352 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);353 return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), std::move(SSP),354 ELFObjFile.getELFFile(),355 std::move(*Features))356 .buildGraph();357}358 359void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,360 std::unique_ptr<JITLinkContext> Ctx) {361 PassConfiguration Config;362 363 if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {364 365 Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));366 Config.PrePrunePasses.push_back(EHFrameEdgeFixer(367 ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64,368 x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32));369 Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));370 371 // Construct a JITLinker and run the link function.372 // Add a mark-live pass.373 if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))374 Config.PrePrunePasses.push_back(std::move(MarkLive));375 else376 Config.PrePrunePasses.push_back(markAllSymbolsLive);377 378 // Add an in-place GOT/Stubs/TLSInfoEntry build pass.379 Config.PostPrunePasses.push_back(buildTables_ELF_x86_64);380 381 // Resolve any external section start / end symbols.382 Config.PostAllocationPasses.push_back(383 createDefineExternalSectionStartAndEndSymbolsPass(384 identifyELFSectionStartAndEndSymbols));385 386 // Add GOT/Stubs optimizer pass.387 Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses);388 }389 390 if (auto Err = Ctx->modifyPassConfig(*G, Config))391 return Ctx->notifyFailed(std::move(Err));392 393 ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));394}395} // end namespace jitlink396} // end namespace llvm397