614 lines · cpp
1//===---------- LazyReexports.cpp - Utilities for lazy reexports ----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ExecutionEngine/Orc/LazyReexports.h"10 11#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"12#include "llvm/ExecutionEngine/Orc/OrcABISupport.h"13#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h"14#include "llvm/TargetParser/Triple.h"15 16#define DEBUG_TYPE "orc"17 18namespace llvm {19namespace orc {20 21LazyCallThroughManager::LazyCallThroughManager(ExecutionSession &ES,22 ExecutorAddr ErrorHandlerAddr,23 TrampolinePool *TP)24 : ES(ES), ErrorHandlerAddr(ErrorHandlerAddr), TP(TP) {}25 26Expected<ExecutorAddr> LazyCallThroughManager::getCallThroughTrampoline(27 JITDylib &SourceJD, SymbolStringPtr SymbolName,28 NotifyResolvedFunction NotifyResolved) {29 assert(TP && "TrampolinePool not set");30 31 std::lock_guard<std::mutex> Lock(LCTMMutex);32 auto Trampoline = TP->getTrampoline();33 34 if (!Trampoline)35 return Trampoline.takeError();36 37 Reexports[*Trampoline] = ReexportsEntry{&SourceJD, std::move(SymbolName)};38 Notifiers[*Trampoline] = std::move(NotifyResolved);39 return *Trampoline;40}41 42ExecutorAddr LazyCallThroughManager::reportCallThroughError(Error Err) {43 ES.reportError(std::move(Err));44 return ErrorHandlerAddr;45}46 47Expected<LazyCallThroughManager::ReexportsEntry>48LazyCallThroughManager::findReexport(ExecutorAddr TrampolineAddr) {49 std::lock_guard<std::mutex> Lock(LCTMMutex);50 auto I = Reexports.find(TrampolineAddr);51 if (I == Reexports.end())52 return createStringError(inconvertibleErrorCode(),53 "Missing reexport for trampoline address %p" +54 formatv("{0:x}", TrampolineAddr));55 return I->second;56}57 58Error LazyCallThroughManager::notifyResolved(ExecutorAddr TrampolineAddr,59 ExecutorAddr ResolvedAddr) {60 NotifyResolvedFunction NotifyResolved;61 {62 std::lock_guard<std::mutex> Lock(LCTMMutex);63 auto I = Notifiers.find(TrampolineAddr);64 if (I != Notifiers.end()) {65 NotifyResolved = std::move(I->second);66 Notifiers.erase(I);67 }68 }69 70 return NotifyResolved ? NotifyResolved(ResolvedAddr) : Error::success();71}72 73void LazyCallThroughManager::resolveTrampolineLandingAddress(74 ExecutorAddr TrampolineAddr,75 NotifyLandingResolvedFunction NotifyLandingResolved) {76 77 auto Entry = findReexport(TrampolineAddr);78 if (!Entry)79 return NotifyLandingResolved(reportCallThroughError(Entry.takeError()));80 81 // Declaring SLS and the callback outside of the call to ES.lookup is a82 // workaround to fix build failures on AIX and on z/OS platforms.83 SymbolLookupSet SLS({Entry->SymbolName});84 auto Callback = [this, TrampolineAddr, SymbolName = Entry->SymbolName,85 NotifyLandingResolved = std::move(NotifyLandingResolved)](86 Expected<SymbolMap> Result) mutable {87 if (Result) {88 assert(Result->size() == 1 && "Unexpected result size");89 assert(Result->count(SymbolName) && "Unexpected result value");90 ExecutorAddr LandingAddr = (*Result)[SymbolName].getAddress();91 92 if (auto Err = notifyResolved(TrampolineAddr, LandingAddr))93 NotifyLandingResolved(reportCallThroughError(std::move(Err)));94 else95 NotifyLandingResolved(LandingAddr);96 } else {97 NotifyLandingResolved(reportCallThroughError(Result.takeError()));98 }99 };100 101 ES.lookup(LookupKind::Static,102 makeJITDylibSearchOrder(Entry->SourceJD,103 JITDylibLookupFlags::MatchAllSymbols),104 std::move(SLS), SymbolState::Ready, std::move(Callback),105 NoDependenciesToRegister);106}107 108Expected<std::unique_ptr<LazyCallThroughManager>>109createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES,110 ExecutorAddr ErrorHandlerAddr) {111 switch (T.getArch()) {112 default:113 return make_error<StringError>(114 std::string("No callback manager available for ") + T.str(),115 inconvertibleErrorCode());116 117 case Triple::aarch64:118 case Triple::aarch64_32:119 return LocalLazyCallThroughManager::Create<OrcAArch64>(ES,120 ErrorHandlerAddr);121 122 case Triple::x86:123 return LocalLazyCallThroughManager::Create<OrcI386>(ES, ErrorHandlerAddr);124 125 case Triple::loongarch64:126 return LocalLazyCallThroughManager::Create<OrcLoongArch64>(127 ES, ErrorHandlerAddr);128 129 case Triple::mips:130 return LocalLazyCallThroughManager::Create<OrcMips32Be>(ES,131 ErrorHandlerAddr);132 133 case Triple::mipsel:134 return LocalLazyCallThroughManager::Create<OrcMips32Le>(ES,135 ErrorHandlerAddr);136 137 case Triple::mips64:138 case Triple::mips64el:139 return LocalLazyCallThroughManager::Create<OrcMips64>(ES, ErrorHandlerAddr);140 141 case Triple::riscv64:142 return LocalLazyCallThroughManager::Create<OrcRiscv64>(ES,143 ErrorHandlerAddr);144 145 case Triple::x86_64:146 if (T.getOS() == Triple::OSType::Win32)147 return LocalLazyCallThroughManager::Create<OrcX86_64_Win32>(148 ES, ErrorHandlerAddr);149 else150 return LocalLazyCallThroughManager::Create<OrcX86_64_SysV>(151 ES, ErrorHandlerAddr);152 }153}154 155LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit(156 LazyCallThroughManager &LCTManager, RedirectableSymbolManager &RSManager,157 JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc)158 : MaterializationUnit(extractFlags(CallableAliases)),159 LCTManager(LCTManager), RSManager(RSManager), SourceJD(SourceJD),160 CallableAliases(std::move(CallableAliases)), AliaseeTable(SrcJDLoc) {}161 162StringRef LazyReexportsMaterializationUnit::getName() const {163 return "<Lazy Reexports>";164}165 166void LazyReexportsMaterializationUnit::materialize(167 std::unique_ptr<MaterializationResponsibility> R) {168 auto RequestedSymbols = R->getRequestedSymbols();169 170 SymbolAliasMap RequestedAliases;171 for (auto &RequestedSymbol : RequestedSymbols) {172 auto I = CallableAliases.find(RequestedSymbol);173 assert(I != CallableAliases.end() && "Symbol not found in alias map?");174 RequestedAliases[I->first] = std::move(I->second);175 CallableAliases.erase(I);176 }177 178 if (!CallableAliases.empty())179 if (auto Err = R->replace(lazyReexports(LCTManager, RSManager, SourceJD,180 std::move(CallableAliases),181 AliaseeTable))) {182 R->getExecutionSession().reportError(std::move(Err));183 R->failMaterialization();184 return;185 }186 187 SymbolMap Inits;188 for (auto &Alias : RequestedAliases) {189 auto CallThroughTrampoline = LCTManager.getCallThroughTrampoline(190 SourceJD, Alias.second.Aliasee,191 [&TargetJD = R->getTargetJITDylib(), &RSManager = this->RSManager,192 StubSym = Alias.first](ExecutorAddr ResolvedAddr) -> Error {193 return RSManager.redirect(TargetJD, StubSym,194 ExecutorSymbolDef(ResolvedAddr, {}));195 });196 197 if (!CallThroughTrampoline) {198 R->getExecutionSession().reportError(CallThroughTrampoline.takeError());199 R->failMaterialization();200 return;201 }202 203 Inits[Alias.first] = {*CallThroughTrampoline, Alias.second.AliasFlags};204 }205 206 if (AliaseeTable != nullptr && !RequestedAliases.empty())207 AliaseeTable->trackImpls(RequestedAliases, &SourceJD);208 209 if (auto Err = R->replace(std::make_unique<RedirectableMaterializationUnit>(210 RSManager, std::move(Inits)))) {211 R->getExecutionSession().reportError(std::move(Err));212 return R->failMaterialization();213 }214}215 216void LazyReexportsMaterializationUnit::discard(const JITDylib &JD,217 const SymbolStringPtr &Name) {218 assert(CallableAliases.count(Name) &&219 "Symbol not covered by this MaterializationUnit");220 CallableAliases.erase(Name);221}222 223MaterializationUnit::Interface224LazyReexportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {225 SymbolFlagsMap SymbolFlags;226 for (auto &KV : Aliases) {227 assert(KV.second.AliasFlags.isCallable() &&228 "Lazy re-exports must be callable symbols");229 SymbolFlags[KV.first] = KV.second.AliasFlags;230 }231 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);232}233 234class LazyReexportsManager::MU : public MaterializationUnit {235public:236 MU(LazyReexportsManager &LRMgr, SymbolAliasMap Reexports)237 : MaterializationUnit(getInterface(Reexports)), LRMgr(LRMgr),238 Reexports(std::move(Reexports)) {}239 240private:241 Interface getInterface(const SymbolAliasMap &Reexports) {242 SymbolFlagsMap SF;243 for (auto &[Alias, AI] : Reexports)244 SF[Alias] = AI.AliasFlags;245 return {std::move(SF), nullptr};246 }247 248 StringRef getName() const override { return "LazyReexportsManager::MU"; }249 250 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {251 LRMgr.emitReentryTrampolines(std::move(R), std::move(Reexports));252 }253 254 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {255 Reexports.erase(Name);256 }257 258 LazyReexportsManager &LRMgr;259 SymbolAliasMap Reexports;260};261 262class LazyReexportsManager::Plugin : public ObjectLinkingLayer::Plugin {263public:264 void modifyPassConfig(MaterializationResponsibility &MR,265 jitlink::LinkGraph &G,266 jitlink::PassConfiguration &Config) override {}267 268 Error notifyFailed(MaterializationResponsibility &MR) override {269 return Error::success();270 }271 272 Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override {273 return Error::success();274 }275 276 void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey,277 ResourceKey SrcKey) override {}278 279private:280 std::mutex M;281};282 283LazyReexportsManager::Listener::~Listener() = default;284 285Expected<std::unique_ptr<LazyReexportsManager>>286LazyReexportsManager::Create(EmitTrampolinesFn EmitTrampolines,287 RedirectableSymbolManager &RSMgr,288 JITDylib &PlatformJD, Listener *L) {289 Error Err = Error::success();290 std::unique_ptr<LazyReexportsManager> LRM(new LazyReexportsManager(291 std::move(EmitTrampolines), RSMgr, PlatformJD, L, Err));292 if (Err)293 return std::move(Err);294 return std::move(LRM);295}296 297Error LazyReexportsManager::handleRemoveResources(JITDylib &JD, ResourceKey K) {298 return JD.getExecutionSession().runSessionLocked([&]() -> Error {299 auto I = KeyToReentryAddrs.find(K);300 if (I == KeyToReentryAddrs.end())301 return Error::success();302 303 auto &ReentryAddrs = I->second;304 for (auto &ReentryAddr : ReentryAddrs) {305 assert(CallThroughs.count(ReentryAddr) && "CallTrhough missing");306 CallThroughs.erase(ReentryAddr);307 }308 KeyToReentryAddrs.erase(I);309 return L ? L->onLazyReexportsRemoved(JD, K) : Error::success();310 });311}312 313void LazyReexportsManager::handleTransferResources(JITDylib &JD,314 ResourceKey DstK,315 ResourceKey SrcK) {316 auto I = KeyToReentryAddrs.find(SrcK);317 if (I != KeyToReentryAddrs.end()) {318 auto J = KeyToReentryAddrs.find(DstK);319 if (J == KeyToReentryAddrs.end()) {320 auto Tmp = std::move(I->second);321 KeyToReentryAddrs.erase(I);322 KeyToReentryAddrs[DstK] = std::move(Tmp);323 } else {324 auto &SrcAddrs = I->second;325 auto &DstAddrs = J->second;326 llvm::append_range(DstAddrs, SrcAddrs);327 KeyToReentryAddrs.erase(I);328 }329 if (L)330 L->onLazyReexportsTransfered(JD, DstK, SrcK);331 }332}333 334LazyReexportsManager::LazyReexportsManager(EmitTrampolinesFn EmitTrampolines,335 RedirectableSymbolManager &RSMgr,336 JITDylib &PlatformJD, Listener *L,337 Error &Err)338 : ES(PlatformJD.getExecutionSession()),339 EmitTrampolines(std::move(EmitTrampolines)), RSMgr(RSMgr), L(L) {340 341 using namespace shared;342 343 ErrorAsOutParameter _(&Err);344 345 ExecutionSession::JITDispatchHandlerAssociationMap WFs;346 347 WFs[ES.intern("__orc_rt_resolve_tag")] =348 ES.wrapAsyncWithSPS<SPSExpected<SPSExecutorSymbolDef>(SPSExecutorAddr)>(349 this, &LazyReexportsManager::resolve);350 351 Err = ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));352}353 354std::unique_ptr<MaterializationUnit>355LazyReexportsManager::createLazyReexports(SymbolAliasMap Reexports) {356 return std::make_unique<MU>(*this, std::move(Reexports));357}358 359void LazyReexportsManager::emitReentryTrampolines(360 std::unique_ptr<MaterializationResponsibility> MR,361 SymbolAliasMap Reexports) {362 size_t NumTrampolines = Reexports.size();363 auto RT = MR->getResourceTracker();364 EmitTrampolines(365 std::move(RT), NumTrampolines,366 [this, MR = std::move(MR), Reexports = std::move(Reexports)](367 Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) mutable {368 emitRedirectableSymbols(std::move(MR), std::move(Reexports),369 std::move(ReentryPoints));370 });371}372 373void LazyReexportsManager::emitRedirectableSymbols(374 std::unique_ptr<MaterializationResponsibility> MR, SymbolAliasMap Reexports,375 Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) {376 377 if (!ReentryPoints) {378 MR->getExecutionSession().reportError(ReentryPoints.takeError());379 MR->failMaterialization();380 return;381 }382 383 assert(Reexports.size() == ReentryPoints->size() &&384 "Number of reentry points doesn't match number of reexports");385 386 // Bind entry points to names.387 SymbolMap Redirs;388 size_t I = 0;389 for (auto &[Name, AI] : Reexports)390 Redirs[Name] = {(*ReentryPoints)[I++].getAddress(), AI.AliasFlags};391 392 I = 0;393 if (!Reexports.empty()) {394 if (auto Err = MR->withResourceKeyDo([&](ResourceKey K) {395 auto &JD = MR->getTargetJITDylib();396 auto &ReentryAddrsForK = KeyToReentryAddrs[K];397 for (auto &[Name, AI] : Reexports) {398 const auto &ReentryPoint = (*ReentryPoints)[I++];399 CallThroughs[ReentryPoint.getAddress()] = {&JD, Name, AI.Aliasee};400 ReentryAddrsForK.push_back(ReentryPoint.getAddress());401 }402 if (L)403 L->onLazyReexportsCreated(JD, K, Reexports);404 })) {405 MR->getExecutionSession().reportError(std::move(Err));406 MR->failMaterialization();407 return;408 }409 }410 411 RSMgr.emitRedirectableSymbols(std::move(MR), std::move(Redirs));412}413 414void LazyReexportsManager::resolve(ResolveSendResultFn SendResult,415 ExecutorAddr ReentryStubAddr) {416 417 CallThroughInfo LandingInfo;418 419 ES.runSessionLocked([&]() {420 auto I = CallThroughs.find(ReentryStubAddr);421 if (I == CallThroughs.end())422 return SendResult(make_error<StringError>(423 "Reentry address " + formatv("{0:x}", ReentryStubAddr) +424 " not registered",425 inconvertibleErrorCode()));426 LandingInfo = I->second;427 });428 429 if (L)430 L->onLazyReexportCalled(LandingInfo);431 432 SymbolInstance LandingSym(LandingInfo.JD, std::move(LandingInfo.BodyName));433 LandingSym.lookupAsync([this, JD = std::move(LandingInfo.JD),434 ReentryName = std::move(LandingInfo.Name),435 SendResult = std::move(SendResult)](436 Expected<ExecutorSymbolDef> Result) mutable {437 if (Result) {438 // FIXME: Make RedirectionManager operations async, then use the async439 // APIs here.440 if (auto Err = RSMgr.redirect(*JD, ReentryName, *Result))441 SendResult(std::move(Err));442 else443 SendResult(std::move(Result));444 } else445 SendResult(std::move(Result));446 });447}448 449class SimpleLazyReexportsSpeculator::SpeculateTask : public IdleTask {450public:451 SpeculateTask(std::weak_ptr<SimpleLazyReexportsSpeculator> Speculator)452 : Speculator(std::move(Speculator)) {}453 454 void printDescription(raw_ostream &OS) override {455 OS << "Speculative Lookup Task";456 }457 458 void run() override {459 if (auto S = Speculator.lock())460 S->doNextSpeculativeLookup();461 }462 463private:464 std::weak_ptr<SimpleLazyReexportsSpeculator> Speculator;465};466 467SimpleLazyReexportsSpeculator::~SimpleLazyReexportsSpeculator() {468 for (auto &[JD, _] : LazyReexports)469 JITDylibSP(JD)->Release();470}471 472void SimpleLazyReexportsSpeculator::onLazyReexportsCreated(473 JITDylib &JD, ResourceKey K, const SymbolAliasMap &Reexports) {474 if (!LazyReexports.count(&JD))475 JD.Retain();476 auto &BodiesVec = LazyReexports[&JD][K];477 for (auto &[Name, AI] : Reexports)478 BodiesVec.push_back(AI.Aliasee);479 if (!SpeculateTaskActive) {480 SpeculateTaskActive = true;481 ES.dispatchTask(std::make_unique<SpeculateTask>(WeakThis));482 }483}484 485void SimpleLazyReexportsSpeculator::onLazyReexportsTransfered(486 JITDylib &JD, ResourceKey DstK, ResourceKey SrcK) {487 488 auto I = LazyReexports.find(&JD);489 if (I == LazyReexports.end())490 return;491 492 auto &MapForJD = I->second;493 auto J = MapForJD.find(SrcK);494 if (J == MapForJD.end())495 return;496 497 // We have something to transfer.498 auto K = MapForJD.find(DstK);499 if (K == MapForJD.end()) {500 auto Tmp = std::move(J->second);501 MapForJD.erase(J);502 MapForJD[DstK] = std::move(Tmp);503 } else {504 auto &SrcNames = J->second;505 auto &DstNames = K->second;506 llvm::append_range(DstNames, SrcNames);507 MapForJD.erase(J);508 }509}510 511Error SimpleLazyReexportsSpeculator::onLazyReexportsRemoved(JITDylib &JD,512 ResourceKey K) {513 514 auto I = LazyReexports.find(&JD);515 if (I == LazyReexports.end())516 return Error::success();517 518 auto &MapForJD = I->second;519 MapForJD.erase(K);520 521 if (MapForJD.empty()) {522 LazyReexports.erase(I);523 JD.Release();524 }525 526 return Error::success();527}528 529void SimpleLazyReexportsSpeculator::onLazyReexportCalled(530 const CallThroughInfo &CTI) {531 if (RecordExec)532 RecordExec(CTI);533}534 535void SimpleLazyReexportsSpeculator::addSpeculationSuggestions(536 std::vector<std::pair<std::string, SymbolStringPtr>> NewSuggestions) {537 ES.runSessionLocked([&]() {538 for (auto &[JDName, SymbolName] : NewSuggestions)539 SpeculateSuggestions.push_back(540 {std::move(JDName), std::move(SymbolName)});541 });542}543 544bool SimpleLazyReexportsSpeculator::doNextSpeculativeLookup() {545 // Use existing speculation queue if available, otherwise take the next546 // element from LazyReexports.547 JITDylibSP SpeculateJD = nullptr;548 SymbolStringPtr SpeculateFn;549 550 auto SpeculateAgain = ES.runSessionLocked([&]() {551 while (!SpeculateSuggestions.empty()) {552 auto [JDName, SymbolName] = std::move(SpeculateSuggestions.front());553 SpeculateSuggestions.pop_front();554 555 if (auto *JD = ES.getJITDylibByName(JDName)) {556 SpeculateJD = JD;557 SpeculateFn = std::move(SymbolName);558 break;559 }560 }561 562 if (!SpeculateJD) {563 assert(!LazyReexports.empty() && "LazyReexports map is empty");564 auto LRItr =565 std::next(LazyReexports.begin(), rand() % LazyReexports.size());566 auto &[JD, KeyToFnBodies] = *LRItr;567 568 assert(!KeyToFnBodies.empty() && "Key to function bodies map empty");569 auto KeyToFnBodiesItr =570 std::next(KeyToFnBodies.begin(), rand() % KeyToFnBodies.size());571 auto &[Key, FnBodies] = *KeyToFnBodiesItr;572 573 assert(!FnBodies.empty() && "Function bodies list empty");574 auto FnBodyItr = std::next(FnBodies.begin(), rand() % FnBodies.size());575 576 SpeculateJD = JITDylibSP(JD);577 SpeculateFn = std::move(*FnBodyItr);578 579 FnBodies.erase(FnBodyItr);580 if (FnBodies.empty()) {581 KeyToFnBodies.erase(KeyToFnBodiesItr);582 if (KeyToFnBodies.empty()) {583 LRItr->first->Release();584 LazyReexports.erase(LRItr);585 }586 }587 }588 589 SpeculateTaskActive =590 !SpeculateSuggestions.empty() || !LazyReexports.empty();591 return SpeculateTaskActive;592 });593 594 LLVM_DEBUG({595 dbgs() << "Issuing speculative lookup for ( " << SpeculateJD->getName()596 << ", " << SpeculateFn << " )...\n";597 });598 599 ES.lookup(600 LookupKind::Static, makeJITDylibSearchOrder(SpeculateJD.get()),601 {{std::move(SpeculateFn), SymbolLookupFlags::WeaklyReferencedSymbol}},602 SymbolState::Ready,603 [](Expected<SymbolMap> Result) { consumeError(Result.takeError()); },604 NoDependenciesToRegister);605 606 if (SpeculateAgain)607 ES.dispatchTask(std::make_unique<SpeculateTask>(WeakThis));608 609 return false;610}611 612} // End namespace orc.613} // End namespace llvm.614