1033 lines · cpp
1//===----- ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc -----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"10 11#include "llvm/ExecutionEngine/JITLink/aarch64.h"12#include "llvm/ExecutionEngine/JITLink/loongarch.h"13#include "llvm/ExecutionEngine/JITLink/ppc64.h"14#include "llvm/ExecutionEngine/JITLink/x86_64.h"15#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"16#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"17#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"18#include "llvm/Support/Debug.h"19#include <optional>20 21#define DEBUG_TYPE "orc"22 23using namespace llvm;24using namespace llvm::orc;25using namespace llvm::orc::shared;26 27namespace {28 29template <typename SPSSerializer, typename... ArgTs>30shared::WrapperFunctionCall::ArgDataBufferType31getArgDataBufferType(const ArgTs &...Args) {32 shared::WrapperFunctionCall::ArgDataBufferType ArgData;33 ArgData.resize(SPSSerializer::size(Args...));34 SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),35 ArgData.size());36 if (SPSSerializer::serialize(OB, Args...))37 return ArgData;38 return {};39}40 41std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,42 std::string Name) {43 auto &ES = MOP.getExecutionSession();44 return std::make_unique<jitlink::LinkGraph>(45 std::move(Name), ES.getSymbolStringPool(), ES.getTargetTriple(),46 SubtargetFeatures(), jitlink::getGenericEdgeKindName);47}48 49// Creates a Bootstrap-Complete LinkGraph to run deferred actions.50class ELFNixPlatformCompleteBootstrapMaterializationUnit51 : public MaterializationUnit {52public:53 ELFNixPlatformCompleteBootstrapMaterializationUnit(54 ELFNixPlatform &MOP, StringRef PlatformJDName,55 SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,56 ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,57 ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,58 ExecutorAddr DeregisterJITDylib)59 : MaterializationUnit(60 {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),61 MOP(MOP), PlatformJDName(PlatformJDName),62 CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),63 DeferredAAsMap(std::move(DeferredAAs)),64 ELFNixHeaderAddr(ELFNixHeaderAddr),65 PlatformBootstrap(PlatformBootstrap),66 PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),67 DeregisterJITDylib(DeregisterJITDylib) {}68 69 StringRef getName() const override {70 return "ELFNixPlatformCompleteBootstrap";71 }72 73 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {74 using namespace jitlink;75 auto G = createPlatformGraph(MOP, "<OrcRTCompleteBootstrap>");76 auto &PlaceholderSection =77 G->createSection("__orc_rt_cplt_bs", MemProt::Read);78 auto &PlaceholderBlock =79 G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0);80 G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1,81 Linkage::Strong, Scope::Hidden, false, true);82 83 // 1. Bootstrap the platform support code.84 G->allocActions().push_back(85 {cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(86 PlatformBootstrap, ELFNixHeaderAddr)),87 cantFail(88 WrapperFunctionCall::Create<SPSArgList<>>(PlatformShutdown))});89 90 // 2. Register the platform JITDylib.91 G->allocActions().push_back(92 {cantFail(WrapperFunctionCall::Create<93 SPSArgList<SPSString, SPSExecutorAddr>>(94 RegisterJITDylib, PlatformJDName, ELFNixHeaderAddr)),95 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(96 DeregisterJITDylib, ELFNixHeaderAddr))});97 98 // 4. Add the deferred actions to the graph.99 for (auto &[Fn, CallDatas] : DeferredAAsMap) {100 for (auto &CallData : CallDatas) {101 G->allocActions().push_back(102 {WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),103 WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});104 }105 }106 107 MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));108 }109 110 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}111 112private:113 ELFNixPlatform &MOP;114 StringRef PlatformJDName;115 SymbolStringPtr CompleteBootstrapSymbol;116 DeferredRuntimeFnMap DeferredAAsMap;117 ExecutorAddr ELFNixHeaderAddr;118 ExecutorAddr PlatformBootstrap;119 ExecutorAddr PlatformShutdown;120 ExecutorAddr RegisterJITDylib;121 ExecutorAddr DeregisterJITDylib;122};123 124class DSOHandleMaterializationUnit : public MaterializationUnit {125public:126 DSOHandleMaterializationUnit(ELFNixPlatform &ENP,127 const SymbolStringPtr &DSOHandleSymbol)128 : MaterializationUnit(129 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),130 ENP(ENP) {}131 132 StringRef getName() const override { return "DSOHandleMU"; }133 134 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {135 136 auto &ES = ENP.getExecutionSession();137 138 jitlink::Edge::Kind EdgeKind;139 140 switch (ES.getTargetTriple().getArch()) {141 case Triple::x86_64:142 EdgeKind = jitlink::x86_64::Pointer64;143 break;144 case Triple::aarch64:145 EdgeKind = jitlink::aarch64::Pointer64;146 break;147 case Triple::ppc64:148 EdgeKind = jitlink::ppc64::Pointer64;149 break;150 case Triple::ppc64le:151 EdgeKind = jitlink::ppc64::Pointer64;152 break;153 case Triple::loongarch64:154 EdgeKind = jitlink::loongarch::Pointer64;155 break;156 default:157 llvm_unreachable("Unrecognized architecture");158 }159 160 // void *__dso_handle = &__dso_handle;161 auto G = std::make_unique<jitlink::LinkGraph>(162 "<DSOHandleMU>", ES.getSymbolStringPool(), ES.getTargetTriple(),163 SubtargetFeatures(), jitlink::getGenericEdgeKindName);164 auto &DSOHandleSection =165 G->createSection(".data.__dso_handle", MemProt::Read);166 auto &DSOHandleBlock = G->createContentBlock(167 DSOHandleSection, getDSOHandleContent(G->getPointerSize()),168 orc::ExecutorAddr(), 8, 0);169 auto &DSOHandleSymbol = G->addDefinedSymbol(170 DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),171 jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);172 DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);173 174 ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));175 }176 177 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}178 179private:180 static MaterializationUnit::Interface181 createDSOHandleSectionInterface(ELFNixPlatform &ENP,182 const SymbolStringPtr &DSOHandleSymbol) {183 SymbolFlagsMap SymbolFlags;184 SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;185 return MaterializationUnit::Interface(std::move(SymbolFlags),186 DSOHandleSymbol);187 }188 189 ArrayRef<char> getDSOHandleContent(size_t PointerSize) {190 static const char Content[8] = {0};191 assert(PointerSize <= sizeof Content);192 return {Content, PointerSize};193 }194 195 ELFNixPlatform &ENP;196};197 198} // end anonymous namespace199 200namespace llvm {201namespace orc {202 203Expected<std::unique_ptr<ELFNixPlatform>>204ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,205 JITDylib &PlatformJD,206 std::unique_ptr<DefinitionGenerator> OrcRuntime,207 std::optional<SymbolAliasMap> RuntimeAliases) {208 209 auto &ES = ObjLinkingLayer.getExecutionSession();210 211 // If the target is not supported then bail out immediately.212 if (!supportedTarget(ES.getTargetTriple()))213 return make_error<StringError>("Unsupported ELFNixPlatform triple: " +214 ES.getTargetTriple().str(),215 inconvertibleErrorCode());216 217 auto &EPC = ES.getExecutorProcessControl();218 219 // Create default aliases if the caller didn't supply any.220 if (!RuntimeAliases) {221 auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);222 if (!StandardRuntimeAliases)223 return StandardRuntimeAliases.takeError();224 RuntimeAliases = std::move(*StandardRuntimeAliases);225 }226 227 // Define the aliases.228 if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))229 return std::move(Err);230 231 // Add JIT-dispatch function support symbols.232 if (auto Err = PlatformJD.define(233 absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),234 {EPC.getJITDispatchInfo().JITDispatchFunction,235 JITSymbolFlags::Exported}},236 {ES.intern("__orc_rt_jit_dispatch_ctx"),237 {EPC.getJITDispatchInfo().JITDispatchContext,238 JITSymbolFlags::Exported}}})))239 return std::move(Err);240 241 // Create the instance.242 Error Err = Error::success();243 auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(244 ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));245 if (Err)246 return std::move(Err);247 return std::move(P);248}249 250Expected<std::unique_ptr<ELFNixPlatform>>251ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,252 JITDylib &PlatformJD, const char *OrcRuntimePath,253 std::optional<SymbolAliasMap> RuntimeAliases) {254 255 // Create a generator for the ORC runtime archive.256 auto OrcRuntimeArchiveGenerator =257 StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);258 if (!OrcRuntimeArchiveGenerator)259 return OrcRuntimeArchiveGenerator.takeError();260 261 return Create(ObjLinkingLayer, PlatformJD,262 std::move(*OrcRuntimeArchiveGenerator),263 std::move(RuntimeAliases));264}265 266Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {267 if (auto Err = JD.define(std::make_unique<DSOHandleMaterializationUnit>(268 *this, DSOHandleSymbol)))269 return Err;270 271 return ES.lookup({&JD}, DSOHandleSymbol).takeError();272}273 274Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {275 std::lock_guard<std::mutex> Lock(PlatformMutex);276 auto I = JITDylibToHandleAddr.find(&JD);277 if (I != JITDylibToHandleAddr.end()) {278 assert(HandleAddrToJITDylib.count(I->second) &&279 "HandleAddrToJITDylib missing entry");280 HandleAddrToJITDylib.erase(I->second);281 JITDylibToHandleAddr.erase(I);282 }283 return Error::success();284}285 286Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,287 const MaterializationUnit &MU) {288 289 auto &JD = RT.getJITDylib();290 const auto &InitSym = MU.getInitializerSymbol();291 if (!InitSym)292 return Error::success();293 294 RegisteredInitSymbols[&JD].add(InitSym,295 SymbolLookupFlags::WeaklyReferencedSymbol);296 LLVM_DEBUG({297 dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym298 << " for MU " << MU.getName() << "\n";299 });300 return Error::success();301}302 303Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {304 llvm_unreachable("Not supported yet");305}306 307static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,308 ArrayRef<std::pair<const char *, const char *>> AL) {309 for (auto &KV : AL) {310 auto AliasName = ES.intern(KV.first);311 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");312 Aliases[std::move(AliasName)] = {ES.intern(KV.second),313 JITSymbolFlags::Exported};314 }315}316 317Expected<SymbolAliasMap>318ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,319 JITDylib &PlatformJD) {320 SymbolAliasMap Aliases;321 addAliases(ES, Aliases, requiredCXXAliases());322 addAliases(ES, Aliases, standardRuntimeUtilityAliases());323 addAliases(ES, Aliases, standardLazyCompilationAliases());324 return Aliases;325}326 327ArrayRef<std::pair<const char *, const char *>>328ELFNixPlatform::requiredCXXAliases() {329 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {330 {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},331 {"atexit", "__orc_rt_elfnix_atexit"}};332 333 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);334}335 336ArrayRef<std::pair<const char *, const char *>>337ELFNixPlatform::standardRuntimeUtilityAliases() {338 static const std::pair<const char *, const char *>339 StandardRuntimeUtilityAliases[] = {340 {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},341 {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},342 {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},343 {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},344 {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},345 {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},346 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};347 348 return ArrayRef<std::pair<const char *, const char *>>(349 StandardRuntimeUtilityAliases);350}351 352ArrayRef<std::pair<const char *, const char *>>353ELFNixPlatform::standardLazyCompilationAliases() {354 static const std::pair<const char *, const char *>355 StandardLazyCompilationAliases[] = {356 {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};357 358 return ArrayRef<std::pair<const char *, const char *>>(359 StandardLazyCompilationAliases);360}361 362bool ELFNixPlatform::supportedTarget(const Triple &TT) {363 switch (TT.getArch()) {364 case Triple::x86_64:365 case Triple::aarch64:366 // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported367 // right now.368 case Triple::ppc64le:369 case Triple::loongarch64:370 return true;371 default:372 return false;373 }374}375 376ELFNixPlatform::ELFNixPlatform(377 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,378 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)379 : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),380 ObjLinkingLayer(ObjLinkingLayer),381 DSOHandleSymbol(ES.intern("__dso_handle")) {382 ErrorAsOutParameter _(Err);383 ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));384 385 PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));386 387 BootstrapInfo BI;388 Bootstrap = &BI;389 390 // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating391 // the platform now), so set it up.392 if (auto E2 = setupJITDylib(PlatformJD)) {393 Err = std::move(E2);394 return;395 }396 397 // Step (2) Request runtime registration functions to trigger398 // materialization..399 if ((Err = ES.lookup(400 makeJITDylibSearchOrder(&PlatformJD),401 SymbolLookupSet(402 {PlatformBootstrap.Name, PlatformShutdown.Name,403 RegisterJITDylib.Name, DeregisterJITDylib.Name,404 RegisterInitSections.Name, DeregisterInitSections.Name,405 RegisterObjectSections.Name,406 DeregisterObjectSections.Name, CreatePThreadKey.Name}))407 .takeError()))408 return;409 410 // Step (3) Wait for any incidental linker work to complete.411 {412 std::unique_lock<std::mutex> Lock(BI.Mutex);413 BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; });414 Bootstrap = nullptr;415 }416 417 // Step (4) Add complete-bootstrap materialization unit and request.418 auto BootstrapCompleteSymbol =419 ES.intern("__orc_rt_elfnix_complete_bootstrap");420 if ((Err = PlatformJD.define(421 std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(422 *this, PlatformJD.getName(), BootstrapCompleteSymbol,423 std::move(BI.DeferredRTFnMap), BI.ELFNixHeaderAddr,424 PlatformBootstrap.Addr, PlatformShutdown.Addr,425 RegisterJITDylib.Addr, DeregisterJITDylib.Addr))))426 return;427 if ((Err = ES.lookup(makeJITDylibSearchOrder(428 &PlatformJD, JITDylibLookupFlags::MatchAllSymbols),429 std::move(BootstrapCompleteSymbol))430 .takeError()))431 return;432 433 // Associate wrapper function tags with JIT-side function implementations.434 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {435 Err = std::move(E2);436 return;437 }438}439 440Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {441 ExecutionSession::JITDispatchHandlerAssociationMap WFs;442 443 using RecordInitializersSPSSig =444 SPSExpected<SPSELFNixJITDylibDepInfoMap>(SPSExecutorAddr);445 WFs[ES.intern("__orc_rt_elfnix_push_initializers_tag")] =446 ES.wrapAsyncWithSPS<RecordInitializersSPSSig>(447 this, &ELFNixPlatform::rt_recordInitializers);448 449 using LookupSymbolSPSSig =450 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);451 WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =452 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,453 &ELFNixPlatform::rt_lookupSymbol);454 455 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));456}457 458void ELFNixPlatform::pushInitializersLoop(459 PushInitializersSendResultFn SendResult, JITDylibSP JD) {460 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;461 DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;462 SmallVector<JITDylib *, 16> Worklist({JD.get()});463 464 ES.runSessionLocked([&]() {465 while (!Worklist.empty()) {466 // FIXME: Check for defunct dylibs.467 468 auto DepJD = Worklist.back();469 Worklist.pop_back();470 471 // If we've already visited this JITDylib on this iteration then continue.472 auto [It, Inserted] = JDDepMap.try_emplace(DepJD);473 if (!Inserted)474 continue;475 476 // Add dep info.477 auto &DM = It->second;478 DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {479 for (auto &KV : O) {480 if (KV.first == DepJD)481 continue;482 DM.push_back(KV.first);483 Worklist.push_back(KV.first);484 }485 });486 487 // Add any registered init symbols.488 auto RISItr = RegisteredInitSymbols.find(DepJD);489 if (RISItr != RegisteredInitSymbols.end()) {490 NewInitSymbols[DepJD] = std::move(RISItr->second);491 RegisteredInitSymbols.erase(RISItr);492 }493 }494 });495 496 // If there are no further init symbols to look up then send the link order497 // (as a list of header addresses) to the caller.498 if (NewInitSymbols.empty()) {499 500 // To make the list intelligible to the runtime we need to convert all501 // JITDylib pointers to their header addresses. Only include JITDylibs502 // that appear in the JITDylibToHandleAddr map (i.e. those that have been503 // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.504 DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;505 HeaderAddrs.reserve(JDDepMap.size());506 {507 std::lock_guard<std::mutex> Lock(PlatformMutex);508 for (auto &KV : JDDepMap) {509 auto I = JITDylibToHandleAddr.find(KV.first);510 if (I != JITDylibToHandleAddr.end())511 HeaderAddrs[KV.first] = I->second;512 }513 }514 515 // Build the dep info map to return.516 ELFNixJITDylibDepInfoMap DIM;517 DIM.reserve(JDDepMap.size());518 for (auto &KV : JDDepMap) {519 auto HI = HeaderAddrs.find(KV.first);520 // Skip unmanaged JITDylibs.521 if (HI == HeaderAddrs.end())522 continue;523 auto H = HI->second;524 ELFNixJITDylibDepInfo DepInfo;525 for (auto &Dep : KV.second) {526 auto HJ = HeaderAddrs.find(Dep);527 if (HJ != HeaderAddrs.end())528 DepInfo.push_back(HJ->second);529 }530 DIM.push_back(std::make_pair(H, std::move(DepInfo)));531 }532 SendResult(DIM);533 return;534 }535 536 // Otherwise issue a lookup and re-run this phase when it completes.537 lookupInitSymbolsAsync(538 [this, SendResult = std::move(SendResult), JD](Error Err) mutable {539 if (Err)540 SendResult(std::move(Err));541 else542 pushInitializersLoop(std::move(SendResult), JD);543 },544 ES, std::move(NewInitSymbols));545}546 547void ELFNixPlatform::rt_recordInitializers(548 PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {549 JITDylibSP JD;550 {551 std::lock_guard<std::mutex> Lock(PlatformMutex);552 auto I = HandleAddrToJITDylib.find(JDHeaderAddr);553 if (I != HandleAddrToJITDylib.end())554 JD = I->second;555 }556 557 LLVM_DEBUG({558 dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";559 if (JD)560 dbgs() << "pushing initializers for " << JD->getName() << "\n";561 else562 dbgs() << "No JITDylib for header address.\n";563 });564 565 if (!JD) {566 SendResult(make_error<StringError>("No JITDylib with header addr " +567 formatv("{0:x}", JDHeaderAddr),568 inconvertibleErrorCode()));569 return;570 }571 572 pushInitializersLoop(std::move(SendResult), JD);573}574 575void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,576 ExecutorAddr Handle,577 StringRef SymbolName) {578 LLVM_DEBUG({579 dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";580 });581 582 JITDylib *JD = nullptr;583 584 {585 std::lock_guard<std::mutex> Lock(PlatformMutex);586 auto I = HandleAddrToJITDylib.find(Handle);587 if (I != HandleAddrToJITDylib.end())588 JD = I->second;589 }590 591 if (!JD) {592 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");593 SendResult(make_error<StringError>("No JITDylib associated with handle " +594 formatv("{0:x}", Handle),595 inconvertibleErrorCode()));596 return;597 }598 599 // Use functor class to work around XL build compiler issue on AIX.600 class RtLookupNotifyComplete {601 public:602 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)603 : SendResult(std::move(SendResult)) {}604 void operator()(Expected<SymbolMap> Result) {605 if (Result) {606 assert(Result->size() == 1 && "Unexpected result map count");607 SendResult(Result->begin()->second.getAddress());608 } else {609 SendResult(Result.takeError());610 }611 }612 613 private:614 SendSymbolAddressFn SendResult;615 };616 617 ES.lookup(618 LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},619 SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,620 RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);621}622 623Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(624 jitlink::LinkGraph &G) {625 // Increment the active graphs count in BootstrapInfo.626 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);627 ++MP.Bootstrap.load()->ActiveGraphs;628 return Error::success();629}630 631Error ELFNixPlatform::ELFNixPlatformPlugin::632 bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {633 // Record bootstrap function names.634 std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {635 {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},636 {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},637 {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},638 {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},639 {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},640 {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},641 {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},642 {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},643 {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},644 {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};645 646 bool RegisterELFNixHeader = false;647 648 for (auto *Sym : G.defined_symbols()) {649 for (auto &RTSym : RuntimeSymbols) {650 if (Sym->hasName() && *Sym->getName() == RTSym.first) {651 if (*RTSym.second)652 return make_error<StringError>(653 "Duplicate " + RTSym.first +654 " detected during ELFNixPlatform bootstrap",655 inconvertibleErrorCode());656 657 if (*Sym->getName() == *MP.DSOHandleSymbol)658 RegisterELFNixHeader = true;659 660 *RTSym.second = Sym->getAddress();661 }662 }663 }664 665 if (RegisterELFNixHeader) {666 // If this graph defines the elfnix header symbol then create the internal667 // mapping between it and PlatformJD.668 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);669 MP.JITDylibToHandleAddr[&MP.PlatformJD] =670 MP.Bootstrap.load()->ELFNixHeaderAddr;671 MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =672 &MP.PlatformJD;673 }674 675 return Error::success();676}677 678Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(679 jitlink::LinkGraph &G) {680 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);681 assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");682 --MP.Bootstrap.load()->ActiveGraphs;683 // Notify Bootstrap->CV while holding the mutex because the mutex is684 // also keeping Bootstrap->CV alive.685 if (MP.Bootstrap.load()->ActiveGraphs == 0)686 MP.Bootstrap.load()->CV.notify_all();687 return Error::success();688}689 690Error ELFNixPlatform::registerPerObjectSections(691 jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,692 bool IsBootstrapping) {693 using SPSRegisterPerObjSectionsArgs =694 SPSArgList<SPSELFPerObjectSectionsToRegister>;695 696 if (LLVM_UNLIKELY(IsBootstrapping)) {697 Bootstrap.load()->addArgumentsToRTFnMap(698 &RegisterObjectSections, &DeregisterObjectSections,699 getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR),700 getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR));701 return Error::success();702 }703 704 G.allocActions().push_back(705 {cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(706 RegisterObjectSections.Addr, POSR)),707 cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(708 DeregisterObjectSections.Addr, POSR))});709 710 return Error::success();711}712 713Expected<uint64_t> ELFNixPlatform::createPThreadKey() {714 if (!CreatePThreadKey.Addr)715 return make_error<StringError>(716 "Attempting to create pthread key in target, but runtime support has "717 "not been loaded yet",718 inconvertibleErrorCode());719 720 Expected<uint64_t> Result(0);721 if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(722 CreatePThreadKey.Addr, Result))723 return std::move(Err);724 return Result;725}726 727void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(728 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,729 jitlink::PassConfiguration &Config) {730 using namespace jitlink;731 732 bool InBootstrapPhase =733 &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;734 735 // If we're in the bootstrap phase then increment the active graphs.736 if (InBootstrapPhase) {737 Config.PrePrunePasses.push_back(738 [this](LinkGraph &G) { return bootstrapPipelineStart(G); });739 Config.PostAllocationPasses.push_back([this](LinkGraph &G) {740 return bootstrapPipelineRecordRuntimeFunctions(G);741 });742 }743 744 // If the initializer symbol is the __dso_handle symbol then just add745 // the DSO handle support passes.746 if (auto InitSymbol = MR.getInitializerSymbol()) {747 if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {748 addDSOHandleSupportPasses(MR, Config);749 // The DSOHandle materialization unit doesn't require any other750 // support, so we can bail out early.751 return;752 }753 754 /// Preserve init sections.755 Config.PrePrunePasses.push_back(756 [this, &MR](jitlink::LinkGraph &G) -> Error {757 if (auto Err = preserveInitSections(G, MR))758 return Err;759 return Error::success();760 });761 }762 763 // Add passes for eh-frame and TLV support.764 addEHAndTLVSupportPasses(MR, Config, InBootstrapPhase);765 766 // If the object contains initializers then add passes to record them.767 Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib(),768 InBootstrapPhase](jitlink::LinkGraph &G) {769 return registerInitSections(G, JD, InBootstrapPhase);770 });771 772 // If we're in the bootstrap phase then steal allocation actions and then773 // decrement the active graphs.774 if (InBootstrapPhase)775 Config.PostFixupPasses.push_back(776 [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });777}778 779void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(780 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {781 782 Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](783 jitlink::LinkGraph &G) -> Error {784 auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {785 return Sym->getName() == MP.DSOHandleSymbol;786 });787 assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");788 {789 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);790 auto HandleAddr = (*I)->getAddress();791 MP.HandleAddrToJITDylib[HandleAddr] = &JD;792 MP.JITDylibToHandleAddr[&JD] = HandleAddr;793 794 G.allocActions().push_back(795 {cantFail(WrapperFunctionCall::Create<796 SPSArgList<SPSString, SPSExecutorAddr>>(797 MP.RegisterJITDylib.Addr, JD.getName(), HandleAddr)),798 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(799 MP.DeregisterJITDylib.Addr, HandleAddr))});800 }801 return Error::success();802 });803}804 805void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(806 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,807 bool IsBootstrapping) {808 809 // Insert TLV lowering at the start of the PostPrunePasses, since we want810 // it to run before GOT/PLT lowering.811 812 // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build813 // pass has done. Because the TLS descriptor need to be allocate in GOT.814 Config.PostPrunePasses.push_back(815 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {816 return fixTLVSectionsAndEdges(G, JD);817 });818 819 // Add a pass to register the final addresses of the eh-frame and TLV sections820 // with the runtime.821 Config.PostFixupPasses.push_back([this, IsBootstrapping](822 jitlink::LinkGraph &G) -> Error {823 ELFPerObjectSectionsToRegister POSR;824 825 if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) {826 jitlink::SectionRange R(*EHFrameSection);827 if (!R.empty())828 POSR.EHFrameSection = R.getRange();829 }830 831 // Get a pointer to the thread data section if there is one. It will be used832 // below.833 jitlink::Section *ThreadDataSection =834 G.findSectionByName(ELFThreadDataSectionName);835 836 // Handle thread BSS section if there is one.837 if (auto *ThreadBSSSection = G.findSectionByName(ELFThreadBSSSectionName)) {838 // If there's already a thread data section in this graph then merge the839 // thread BSS section content into it, otherwise just treat the thread840 // BSS section as the thread data section.841 if (ThreadDataSection)842 G.mergeSections(*ThreadDataSection, *ThreadBSSSection);843 else844 ThreadDataSection = ThreadBSSSection;845 }846 847 // Having merged thread BSS (if present) and thread data (if present),848 // record the resulting section range.849 if (ThreadDataSection) {850 jitlink::SectionRange R(*ThreadDataSection);851 if (!R.empty())852 POSR.ThreadDataSection = R.getRange();853 }854 855 if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {856 if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))857 return Err;858 }859 860 return Error::success();861 });862}863 864Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(865 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {866 867 if (const auto &InitSymName = MR.getInitializerSymbol()) {868 869 jitlink::Symbol *InitSym = nullptr;870 871 for (auto &InitSection : G.sections()) {872 // Skip non-init sections.873 if (!isELFInitializerSection(InitSection.getName()) ||874 InitSection.empty())875 continue;876 877 // Create the init symbol if it has not been created already and attach it878 // to the first block.879 if (!InitSym) {880 auto &B = **InitSection.blocks().begin();881 InitSym = &G.addDefinedSymbol(882 B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,883 jitlink::Scope::SideEffectsOnly, false, true);884 }885 886 // Add keep-alive edges to anonymous symbols in all other init blocks.887 for (auto *B : InitSection.blocks()) {888 if (B == &InitSym->getBlock())889 continue;890 891 auto &S = G.addAnonymousSymbol(*B, 0, B->getSize(), false, true);892 InitSym->getBlock().addEdge(jitlink::Edge::KeepAlive, 0, S, 0);893 }894 }895 }896 897 return Error::success();898}899 900Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(901 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {902 SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;903 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");904 905 SmallVector<jitlink::Section *> OrderedInitSections;906 for (auto &Sec : G.sections())907 if (isELFInitializerSection(Sec.getName()))908 OrderedInitSections.push_back(&Sec);909 910 // FIXME: This handles priority order within the current graph, but we'll need911 // to include priority information in the initializer allocation912 // actions in order to respect the ordering across multiple graphs.913 llvm::sort(OrderedInitSections, [](const jitlink::Section *LHS,914 const jitlink::Section *RHS) {915 if (LHS->getName().starts_with(".init_array")) {916 if (RHS->getName().starts_with(".init_array")) {917 StringRef LHSPrioStr(LHS->getName());918 StringRef RHSPrioStr(RHS->getName());919 uint64_t LHSPriority;920 bool LHSHasPriority = LHSPrioStr.consume_front(".init_array.") &&921 !LHSPrioStr.getAsInteger(10, LHSPriority);922 uint64_t RHSPriority;923 bool RHSHasPriority = RHSPrioStr.consume_front(".init_array.") &&924 !RHSPrioStr.getAsInteger(10, RHSPriority);925 if (LHSHasPriority)926 return RHSHasPriority ? LHSPriority < RHSPriority : true;927 else if (RHSHasPriority)928 return false;929 // If we get here we'll fall through to the930 // LHS->getName() < RHS->getName() test below.931 } else {932 // .init_array[.N] comes before any non-.init_array[.N] section.933 return true;934 }935 }936 return LHS->getName() < RHS->getName();937 });938 939 for (auto &Sec : OrderedInitSections)940 ELFNixPlatformSecs.push_back(jitlink::SectionRange(*Sec).getRange());941 942 // Dump the scraped inits.943 LLVM_DEBUG({944 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";945 for (auto &Sec : G.sections()) {946 jitlink::SectionRange R(Sec);947 dbgs() << " " << Sec.getName() << ": " << R.getRange() << "\n";948 }949 });950 951 ExecutorAddr HeaderAddr;952 {953 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);954 auto I = MP.JITDylibToHandleAddr.find(&JD);955 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");956 assert(I->second && "Null header registered for JD");957 HeaderAddr = I->second;958 }959 960 using SPSRegisterInitSectionsArgs =961 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;962 963 if (LLVM_UNLIKELY(IsBootstrapping)) {964 MP.Bootstrap.load()->addArgumentsToRTFnMap(965 &MP.RegisterInitSections, &MP.DeregisterInitSections,966 getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,967 ELFNixPlatformSecs),968 getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,969 ELFNixPlatformSecs));970 return Error::success();971 }972 973 G.allocActions().push_back(974 {cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(975 MP.RegisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs)),976 cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(977 MP.DeregisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs))});978 979 return Error::success();980}981 982Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(983 jitlink::LinkGraph &G, JITDylib &JD) {984 auto TLSGetAddrSymbolName = G.intern("__tls_get_addr");985 auto TLSDescResolveSymbolName = G.intern("__tlsdesc_resolver");986 for (auto *Sym : G.external_symbols()) {987 if (Sym->getName() == TLSGetAddrSymbolName) {988 auto TLSGetAddr =989 MP.getExecutionSession().intern("___orc_rt_elfnix_tls_get_addr");990 Sym->setName(std::move(TLSGetAddr));991 } else if (Sym->getName() == TLSDescResolveSymbolName) {992 auto TLSGetAddr =993 MP.getExecutionSession().intern("___orc_rt_elfnix_tlsdesc_resolver");994 Sym->setName(std::move(TLSGetAddr));995 }996 }997 998 auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");999 1000 if (TLSInfoEntrySection) {1001 std::optional<uint64_t> Key;1002 {1003 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);1004 auto I = MP.JITDylibToPThreadKey.find(&JD);1005 if (I != MP.JITDylibToPThreadKey.end())1006 Key = I->second;1007 }1008 if (!Key) {1009 if (auto KeyOrErr = MP.createPThreadKey())1010 Key = *KeyOrErr;1011 else1012 return KeyOrErr.takeError();1013 }1014 1015 uint64_t PlatformKeyBits =1016 support::endian::byte_swap(*Key, G.getEndianness());1017 1018 for (auto *B : TLSInfoEntrySection->blocks()) {1019 // FIXME: The TLS descriptor byte length may different with different1020 // ISA1021 assert(B->getSize() == (G.getPointerSize() * 2) &&1022 "TLS descriptor must be 2 words length");1023 auto TLSInfoEntryContent = B->getMutableContent(G);1024 memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());1025 }1026 }1027 1028 return Error::success();1029}1030 1031} // End namespace orc.1032} // End namespace llvm.1033