912 lines · cpp
1//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ExecutionEngine/Orc/COFFPlatform.h"10 11#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"12#include "llvm/ExecutionEngine/Orc/COFF.h"13#include "llvm/ExecutionEngine/Orc/DebugUtils.h"14#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"15#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"16#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"17 18#include "llvm/Object/COFF.h"19 20#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"21 22#include "llvm/ExecutionEngine/JITLink/x86_64.h"23 24#define DEBUG_TYPE "orc"25 26using namespace llvm;27using namespace llvm::orc;28using namespace llvm::orc::shared;29 30namespace llvm {31namespace orc {32namespace shared {33 34using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;35using SPSCOFFJITDylibDepInfoMap =36 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;37using SPSCOFFObjectSectionsMap =38 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;39using SPSCOFFRegisterObjectSectionsArgs =40 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;41using SPSCOFFDeregisterObjectSectionsArgs =42 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;43 44} // namespace shared45} // namespace orc46} // namespace llvm47namespace {48 49class COFFHeaderMaterializationUnit : public MaterializationUnit {50public:51 COFFHeaderMaterializationUnit(COFFPlatform &CP,52 const SymbolStringPtr &HeaderStartSymbol)53 : MaterializationUnit(createHeaderInterface(CP, HeaderStartSymbol)),54 CP(CP) {}55 56 StringRef getName() const override { return "COFFHeaderMU"; }57 58 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {59 auto G = std::make_unique<jitlink::LinkGraph>(60 "<COFFHeaderMU>", CP.getExecutionSession().getSymbolStringPool(),61 CP.getExecutionSession().getTargetTriple(), SubtargetFeatures(),62 jitlink::getGenericEdgeKindName);63 auto &HeaderSection = G->createSection("__header", MemProt::Read);64 auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);65 66 // Init symbol is __ImageBase symbol.67 auto &ImageBaseSymbol = G->addDefinedSymbol(68 HeaderBlock, 0, *R->getInitializerSymbol(), HeaderBlock.getSize(),69 jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);70 71 addImageBaseRelocationEdge(HeaderBlock, ImageBaseSymbol);72 73 CP.getObjectLinkingLayer().emit(std::move(R), std::move(G));74 }75 76 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}77 78private:79 struct HeaderSymbol {80 const char *Name;81 uint64_t Offset;82 };83 84 struct NTHeader {85 support::ulittle32_t PEMagic;86 object::coff_file_header FileHeader;87 struct PEHeader {88 object::pe32plus_header Header;89 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];90 } OptionalHeader;91 };92 93 struct HeaderBlockContent {94 object::dos_header DOSHeader;95 COFFHeaderMaterializationUnit::NTHeader NTHeader;96 };97 98 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,99 jitlink::Section &HeaderSection) {100 HeaderBlockContent Hdr = {};101 102 // Set up magic103 Hdr.DOSHeader.Magic[0] = 'M';104 Hdr.DOSHeader.Magic[1] = 'Z';105 Hdr.DOSHeader.AddressOfNewExeHeader =106 offsetof(HeaderBlockContent, NTHeader);107 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);108 Hdr.NTHeader.PEMagic = PEMagic;109 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;110 111 switch (G.getTargetTriple().getArch()) {112 case Triple::x86_64:113 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;114 break;115 default:116 llvm_unreachable("Unrecognized architecture");117 }118 119 auto HeaderContent = G.allocateContent(120 ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));121 122 return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8,123 0);124 }125 126 static void addImageBaseRelocationEdge(jitlink::Block &B,127 jitlink::Symbol &ImageBase) {128 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +129 offsetof(NTHeader, OptionalHeader) +130 offsetof(object::pe32plus_header, ImageBase);131 B.addEdge(jitlink::x86_64::Pointer64, ImageBaseOffset, ImageBase, 0);132 }133 134 static MaterializationUnit::Interface135 createHeaderInterface(COFFPlatform &MOP,136 const SymbolStringPtr &HeaderStartSymbol) {137 SymbolFlagsMap HeaderSymbolFlags;138 139 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;140 141 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),142 HeaderStartSymbol);143 }144 145 COFFPlatform &CP;146};147 148} // end anonymous namespace149 150namespace llvm {151namespace orc {152 153Expected<std::unique_ptr<COFFPlatform>>154COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,155 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,156 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,157 const char *VCRuntimePath,158 std::optional<SymbolAliasMap> RuntimeAliases) {159 160 auto &ES = ObjLinkingLayer.getExecutionSession();161 162 // If the target is not supported then bail out immediately.163 if (!supportedTarget(ES.getTargetTriple()))164 return make_error<StringError>("Unsupported COFFPlatform triple: " +165 ES.getTargetTriple().str(),166 inconvertibleErrorCode());167 168 auto &EPC = ES.getExecutorProcessControl();169 170 auto GeneratorArchive =171 object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef());172 if (!GeneratorArchive)173 return GeneratorArchive.takeError();174 175 std::set<std::string> DylibsToPreload;176 auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(177 ObjLinkingLayer, nullptr, std::move(*GeneratorArchive),178 COFFImportFileScanner(DylibsToPreload));179 if (!OrcRuntimeArchiveGenerator)180 return OrcRuntimeArchiveGenerator.takeError();181 182 // We need a second instance of the archive (for now) for the Platform. We183 // can `cantFail` this call, since if it were going to fail it would have184 // failed above.185 auto RuntimeArchive = cantFail(186 object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef()));187 188 // Create default aliases if the caller didn't supply any.189 if (!RuntimeAliases)190 RuntimeAliases = standardPlatformAliases(ES);191 192 // Define the aliases.193 if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))194 return std::move(Err);195 196 auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>");197 198 // Add JIT-dispatch function support symbols.199 if (auto Err = HostFuncJD.define(200 absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),201 {EPC.getJITDispatchInfo().JITDispatchFunction,202 JITSymbolFlags::Exported}},203 {ES.intern("__orc_rt_jit_dispatch_ctx"),204 {EPC.getJITDispatchInfo().JITDispatchContext,205 JITSymbolFlags::Exported}}})))206 return std::move(Err);207 208 PlatformJD.addToLinkOrder(HostFuncJD);209 210 // Create the instance.211 Error Err = Error::success();212 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(213 ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),214 std::move(DylibsToPreload), std::move(OrcRuntimeArchiveBuffer),215 std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime,216 VCRuntimePath, Err));217 if (Err)218 return std::move(Err);219 return std::move(P);220}221 222Expected<std::unique_ptr<COFFPlatform>>223COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,224 const char *OrcRuntimePath,225 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,226 const char *VCRuntimePath,227 std::optional<SymbolAliasMap> RuntimeAliases) {228 229 auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath);230 if (!ArchiveBuffer)231 return createFileError(OrcRuntimePath, ArchiveBuffer.getError());232 233 return Create(ObjLinkingLayer, PlatformJD, std::move(*ArchiveBuffer),234 std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,235 std::move(RuntimeAliases));236}237 238Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {239 auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker");240 if (!PerJDObj)241 return PerJDObj.takeError();242 243 if (!*PerJDObj)244 return make_error<StringError>("Could not find per jd object file",245 inconvertibleErrorCode());246 247 auto Buffer = (*PerJDObj)->getAsBinary();248 if (!Buffer)249 return Buffer.takeError();250 251 return (*Buffer)->getMemoryBufferRef();252}253 254static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,255 ArrayRef<std::pair<const char *, const char *>> AL) {256 for (auto &KV : AL) {257 auto AliasName = ES.intern(KV.first);258 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");259 Aliases[std::move(AliasName)] = {ES.intern(KV.second),260 JITSymbolFlags::Exported};261 }262}263 264Error COFFPlatform::setupJITDylib(JITDylib &JD) {265 if (auto Err = JD.define(std::make_unique<COFFHeaderMaterializationUnit>(266 *this, COFFHeaderStartSymbol)))267 return Err;268 269 if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError())270 return Err;271 272 // Define the CXX aliases.273 SymbolAliasMap CXXAliases;274 addAliases(ES, CXXAliases, requiredCXXAliases());275 if (auto Err = JD.define(symbolAliases(std::move(CXXAliases))))276 return Err;277 278 auto PerJDObj = getPerJDObjectFile();279 if (!PerJDObj)280 return PerJDObj.takeError();281 282 auto I = getObjectFileInterface(ES, *PerJDObj);283 if (!I)284 return I.takeError();285 286 if (auto Err = ObjLinkingLayer.add(287 JD, MemoryBuffer::getMemBuffer(*PerJDObj, false), std::move(*I)))288 return Err;289 290 if (!Bootstrapping) {291 auto ImportedLibs = StaticVCRuntime292 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)293 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);294 if (!ImportedLibs)295 return ImportedLibs.takeError();296 for (auto &Lib : *ImportedLibs)297 if (auto Err = LoadDynLibrary(JD, Lib))298 return Err;299 if (StaticVCRuntime)300 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))301 return Err;302 }303 304 JD.addGenerator(DLLImportDefinitionGenerator::Create(ES, ObjLinkingLayer));305 return Error::success();306}307 308Error COFFPlatform::teardownJITDylib(JITDylib &JD) {309 std::lock_guard<std::mutex> Lock(PlatformMutex);310 auto I = JITDylibToHeaderAddr.find(&JD);311 if (I != JITDylibToHeaderAddr.end()) {312 assert(HeaderAddrToJITDylib.count(I->second) &&313 "HeaderAddrToJITDylib missing entry");314 HeaderAddrToJITDylib.erase(I->second);315 JITDylibToHeaderAddr.erase(I);316 }317 return Error::success();318}319 320Error COFFPlatform::notifyAdding(ResourceTracker &RT,321 const MaterializationUnit &MU) {322 auto &JD = RT.getJITDylib();323 const auto &InitSym = MU.getInitializerSymbol();324 if (!InitSym)325 return Error::success();326 327 RegisteredInitSymbols[&JD].add(InitSym,328 SymbolLookupFlags::WeaklyReferencedSymbol);329 330 LLVM_DEBUG({331 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "332 << MU.getName() << "\n";333 });334 return Error::success();335}336 337Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {338 llvm_unreachable("Not supported yet");339}340 341SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {342 SymbolAliasMap Aliases;343 addAliases(ES, Aliases, standardRuntimeUtilityAliases());344 return Aliases;345}346 347ArrayRef<std::pair<const char *, const char *>>348COFFPlatform::requiredCXXAliases() {349 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {350 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},351 {"_onexit", "__orc_rt_coff_onexit_per_jd"},352 {"atexit", "__orc_rt_coff_atexit_per_jd"}};353 354 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);355}356 357ArrayRef<std::pair<const char *, const char *>>358COFFPlatform::standardRuntimeUtilityAliases() {359 static const std::pair<const char *, const char *>360 StandardRuntimeUtilityAliases[] = {361 {"__orc_rt_run_program", "__orc_rt_coff_run_program"},362 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},363 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},364 {"__orc_rt_jit_dlupdate", "__orc_rt_coff_jit_dlupdate"},365 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},366 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},367 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};368 369 return ArrayRef<std::pair<const char *, const char *>>(370 StandardRuntimeUtilityAliases);371}372 373bool COFFPlatform::supportedTarget(const Triple &TT) {374 switch (TT.getArch()) {375 case Triple::x86_64:376 return true;377 default:378 return false;379 }380}381 382COFFPlatform::COFFPlatform(383 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,384 std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,385 std::set<std::string> DylibsToPreload,386 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,387 std::unique_ptr<object::Archive> OrcRuntimeArchive,388 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,389 const char *VCRuntimePath, Error &Err)390 : ES(ObjLinkingLayer.getExecutionSession()),391 ObjLinkingLayer(ObjLinkingLayer),392 LoadDynLibrary(std::move(LoadDynLibrary)),393 OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),394 OrcRuntimeArchive(std::move(OrcRuntimeArchive)),395 StaticVCRuntime(StaticVCRuntime),396 COFFHeaderStartSymbol(ES.intern("__ImageBase")) {397 ErrorAsOutParameter _(Err);398 399 Bootstrapping.store(true);400 ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this));401 402 // Load vc runtime403 auto VCRT =404 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath);405 if (!VCRT) {406 Err = VCRT.takeError();407 return;408 }409 VCRuntimeBootstrap = std::move(*VCRT);410 411 auto ImportedLibs =412 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD)413 : VCRuntimeBootstrap->loadDynamicVCRuntime(PlatformJD);414 if (!ImportedLibs) {415 Err = ImportedLibs.takeError();416 return;417 }418 419 for (auto &Lib : *ImportedLibs)420 DylibsToPreload.insert(Lib);421 422 PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));423 424 // PlatformJD hasn't been set up by the platform yet (since we're creating425 // the platform now), so set it up.426 if (auto E2 = setupJITDylib(PlatformJD)) {427 Err = std::move(E2);428 return;429 }430 431 for (auto& Lib : DylibsToPreload)432 if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {433 Err = std::move(E2);434 return;435 }436 437 if (StaticVCRuntime)438 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) {439 Err = std::move(E2);440 return;441 }442 443 // Associate wrapper function tags with JIT-side function implementations.444 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {445 Err = std::move(E2);446 return;447 }448 449 // Lookup addresses of runtime functions callable by the platform,450 // call the platform bootstrap function to initialize the platform-state451 // object in the executor.452 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {453 Err = std::move(E2);454 return;455 }456 457 Bootstrapping.store(false);458 JDBootstrapStates.clear();459}460 461Expected<COFFPlatform::JITDylibDepMap>462COFFPlatform::buildJDDepMap(JITDylib &JD) {463 return ES.runSessionLocked([&]() -> Expected<JITDylibDepMap> {464 JITDylibDepMap JDDepMap;465 466 SmallVector<JITDylib *, 16> Worklist({&JD});467 while (!Worklist.empty()) {468 auto CurJD = Worklist.back();469 Worklist.pop_back();470 471 auto &DM = JDDepMap[CurJD];472 CurJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {473 DM.reserve(O.size());474 for (auto &KV : O) {475 if (KV.first == CurJD)476 continue;477 {478 // Bare jitdylibs not known to the platform479 std::lock_guard<std::mutex> Lock(PlatformMutex);480 if (!JITDylibToHeaderAddr.count(KV.first)) {481 LLVM_DEBUG({482 dbgs() << "JITDylib unregistered to COFFPlatform detected in "483 "LinkOrder: "484 << CurJD->getName() << "\n";485 });486 continue;487 }488 }489 DM.push_back(KV.first);490 // Push unvisited entry.491 if (JDDepMap.try_emplace(KV.first).second)492 Worklist.push_back(KV.first);493 }494 });495 }496 return std::move(JDDepMap);497 });498}499 500void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,501 JITDylibSP JD,502 JITDylibDepMap &JDDepMap) {503 SmallVector<JITDylib *, 16> Worklist({JD.get()});504 DenseSet<JITDylib *> Visited({JD.get()});505 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;506 ES.runSessionLocked([&]() {507 while (!Worklist.empty()) {508 auto CurJD = Worklist.back();509 Worklist.pop_back();510 511 auto RISItr = RegisteredInitSymbols.find(CurJD);512 if (RISItr != RegisteredInitSymbols.end()) {513 NewInitSymbols[CurJD] = std::move(RISItr->second);514 RegisteredInitSymbols.erase(RISItr);515 }516 517 for (auto *DepJD : JDDepMap[CurJD])518 if (Visited.insert(DepJD).second)519 Worklist.push_back(DepJD);520 }521 });522 523 // If there are no further init symbols to look up then send the link order524 // (as a list of header addresses) to the caller.525 if (NewInitSymbols.empty()) {526 // Build the dep info map to return.527 COFFJITDylibDepInfoMap DIM;528 DIM.reserve(JDDepMap.size());529 for (auto &KV : JDDepMap) {530 std::lock_guard<std::mutex> Lock(PlatformMutex);531 COFFJITDylibDepInfo DepInfo;532 DepInfo.reserve(KV.second.size());533 for (auto &Dep : KV.second) {534 DepInfo.push_back(JITDylibToHeaderAddr[Dep]);535 }536 auto H = JITDylibToHeaderAddr[KV.first];537 DIM.push_back(std::make_pair(H, std::move(DepInfo)));538 }539 SendResult(DIM);540 return;541 }542 543 // Otherwise issue a lookup and re-run this phase when it completes.544 lookupInitSymbolsAsync(545 [this, SendResult = std::move(SendResult), &JD,546 JDDepMap = std::move(JDDepMap)](Error Err) mutable {547 if (Err)548 SendResult(std::move(Err));549 else550 pushInitializersLoop(std::move(SendResult), JD, JDDepMap);551 },552 ES, std::move(NewInitSymbols));553}554 555void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,556 ExecutorAddr JDHeaderAddr) {557 JITDylibSP JD;558 {559 std::lock_guard<std::mutex> Lock(PlatformMutex);560 auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);561 if (I != HeaderAddrToJITDylib.end())562 JD = I->second;563 }564 565 LLVM_DEBUG({566 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";567 if (JD)568 dbgs() << "pushing initializers for " << JD->getName() << "\n";569 else570 dbgs() << "No JITDylib for header address.\n";571 });572 573 if (!JD) {574 SendResult(make_error<StringError>("No JITDylib with header addr " +575 formatv("{0:x}", JDHeaderAddr),576 inconvertibleErrorCode()));577 return;578 }579 580 auto JDDepMap = buildJDDepMap(*JD);581 if (!JDDepMap) {582 SendResult(JDDepMap.takeError());583 return;584 }585 586 pushInitializersLoop(std::move(SendResult), JD, *JDDepMap);587}588 589void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,590 ExecutorAddr Handle, StringRef SymbolName) {591 LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");592 593 JITDylib *JD = nullptr;594 595 {596 std::lock_guard<std::mutex> Lock(PlatformMutex);597 auto I = HeaderAddrToJITDylib.find(Handle);598 if (I != HeaderAddrToJITDylib.end())599 JD = I->second;600 }601 602 if (!JD) {603 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");604 SendResult(make_error<StringError>("No JITDylib associated with handle " +605 formatv("{0:x}", Handle),606 inconvertibleErrorCode()));607 return;608 }609 610 // Use functor class to work around XL build compiler issue on AIX.611 class RtLookupNotifyComplete {612 public:613 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)614 : SendResult(std::move(SendResult)) {}615 void operator()(Expected<SymbolMap> Result) {616 if (Result) {617 assert(Result->size() == 1 && "Unexpected result map count");618 SendResult(Result->begin()->second.getAddress());619 } else {620 SendResult(Result.takeError());621 }622 }623 624 private:625 SendSymbolAddressFn SendResult;626 };627 628 ES.lookup(629 LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},630 SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,631 RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);632}633 634Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {635 ExecutionSession::JITDispatchHandlerAssociationMap WFs;636 637 using LookupSymbolSPSSig =638 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);639 WFs[ES.intern("__orc_rt_coff_symbol_lookup_tag")] =640 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,641 &COFFPlatform::rt_lookupSymbol);642 using PushInitializersSPSSig =643 SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);644 WFs[ES.intern("__orc_rt_coff_push_initializers_tag")] =645 ES.wrapAsyncWithSPS<PushInitializersSPSSig>(646 this, &COFFPlatform::rt_pushInitializers);647 648 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));649}650 651Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {652 llvm::sort(BState.Initializers);653 if (auto Err =654 runBootstrapSubsectionInitializers(BState, ".CRT$XIA", ".CRT$XIZ"))655 return Err;656 657 if (auto Err = runSymbolIfExists(*BState.JD, "__run_after_c_init"))658 return Err;659 660 if (auto Err =661 runBootstrapSubsectionInitializers(BState, ".CRT$XCA", ".CRT$XCZ"))662 return Err;663 return Error::success();664}665 666Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,667 StringRef Start,668 StringRef End) {669 for (auto &Initializer : BState.Initializers)670 if (Initializer.first >= Start && Initializer.first <= End &&671 Initializer.second) {672 auto Res =673 ES.getExecutorProcessControl().runAsVoidFunction(Initializer.second);674 if (!Res)675 return Res.takeError();676 }677 return Error::success();678}679 680Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {681 // Lookup of runtime symbols causes the collection of initializers if682 // it's static linking setting.683 if (auto Err = lookupAndRecordAddrs(684 ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),685 {686 {ES.intern("__orc_rt_coff_platform_bootstrap"),687 &orc_rt_coff_platform_bootstrap},688 {ES.intern("__orc_rt_coff_platform_shutdown"),689 &orc_rt_coff_platform_shutdown},690 {ES.intern("__orc_rt_coff_register_jitdylib"),691 &orc_rt_coff_register_jitdylib},692 {ES.intern("__orc_rt_coff_deregister_jitdylib"),693 &orc_rt_coff_deregister_jitdylib},694 {ES.intern("__orc_rt_coff_register_object_sections"),695 &orc_rt_coff_register_object_sections},696 {ES.intern("__orc_rt_coff_deregister_object_sections"),697 &orc_rt_coff_deregister_object_sections},698 }))699 return Err;700 701 // Call bootstrap functions702 if (auto Err = ES.callSPSWrapper<void()>(orc_rt_coff_platform_bootstrap))703 return Err;704 705 // Do the pending jitdylib registration actions that we couldn't do706 // because orc runtime was not linked fully.707 for (auto KV : JDBootstrapStates) {708 auto &JDBState = KV.second;709 if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(710 orc_rt_coff_register_jitdylib, JDBState.JDName,711 JDBState.HeaderAddr))712 return Err;713 714 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)715 if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,716 SPSCOFFObjectSectionsMap, bool)>(717 orc_rt_coff_register_object_sections, JDBState.HeaderAddr,718 ObjSectionMap, false))719 return Err;720 }721 722 // Run static initializers collected in bootstrap stage.723 for (auto KV : JDBootstrapStates) {724 auto &JDBState = KV.second;725 if (auto Err = runBootstrapInitializers(JDBState))726 return Err;727 }728 729 return Error::success();730}731 732Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,733 StringRef SymbolName) {734 ExecutorAddr jit_function;735 auto AfterCLookupErr = lookupAndRecordAddrs(736 ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),737 {{ES.intern(SymbolName), &jit_function}});738 if (!AfterCLookupErr) {739 auto Res = ES.getExecutorProcessControl().runAsVoidFunction(jit_function);740 if (!Res)741 return Res.takeError();742 return Error::success();743 }744 if (!AfterCLookupErr.isA<SymbolsNotFound>())745 return AfterCLookupErr;746 consumeError(std::move(AfterCLookupErr));747 return Error::success();748}749 750void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(751 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,752 jitlink::PassConfiguration &Config) {753 754 bool IsBootstrapping = CP.Bootstrapping.load();755 756 if (auto InitSymbol = MR.getInitializerSymbol()) {757 if (InitSymbol == CP.COFFHeaderStartSymbol) {758 Config.PostAllocationPasses.push_back(759 [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {760 return associateJITDylibHeaderSymbol(G, MR, IsBootstrapping);761 });762 return;763 }764 Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) {765 return preserveInitializerSections(G, MR);766 });767 }768 769 if (!IsBootstrapping)770 Config.PostFixupPasses.push_back(771 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {772 return registerObjectPlatformSections(G, JD);773 });774 else775 Config.PostFixupPasses.push_back(776 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {777 return registerObjectPlatformSectionsInBootstrap(G, JD);778 });779}780 781Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(782 jitlink::LinkGraph &G, MaterializationResponsibility &MR,783 bool IsBootstraping) {784 auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {785 return *Sym->getName() == *CP.COFFHeaderStartSymbol;786 });787 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");788 789 auto &JD = MR.getTargetJITDylib();790 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);791 auto HeaderAddr = (*I)->getAddress();792 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;793 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;794 if (!IsBootstraping) {795 G.allocActions().push_back(796 {cantFail(WrapperFunctionCall::Create<797 SPSArgList<SPSString, SPSExecutorAddr>>(798 CP.orc_rt_coff_register_jitdylib, JD.getName(), HeaderAddr)),799 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(800 CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});801 } else {802 G.allocActions().push_back(803 {{},804 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(805 CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});806 JDBootstrapState BState;807 BState.JD = &JD;808 BState.JDName = JD.getName();809 BState.HeaderAddr = HeaderAddr;810 CP.JDBootstrapStates.emplace(&JD, BState);811 }812 813 return Error::success();814}815 816Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(817 jitlink::LinkGraph &G, JITDylib &JD) {818 COFFObjectSectionsMap ObjSecs;819 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];820 assert(HeaderAddr && "Must be registered jitdylib");821 for (auto &S : G.sections()) {822 jitlink::SectionRange Range(S);823 if (Range.getSize())824 ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));825 }826 827 G.allocActions().push_back(828 {cantFail(WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(829 CP.orc_rt_coff_register_object_sections, HeaderAddr, ObjSecs, true)),830 cantFail(831 WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(832 CP.orc_rt_coff_deregister_object_sections, HeaderAddr,833 ObjSecs))});834 835 return Error::success();836}837 838Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(839 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {840 841 if (const auto &InitSymName = MR.getInitializerSymbol()) {842 843 jitlink::Symbol *InitSym = nullptr;844 845 for (auto &InitSection : G.sections()) {846 // Skip non-init sections.847 if (!isCOFFInitializerSection(InitSection.getName()) ||848 InitSection.empty())849 continue;850 851 // Create the init symbol if it has not been created already and attach it852 // to the first block.853 if (!InitSym) {854 auto &B = **InitSection.blocks().begin();855 InitSym = &G.addDefinedSymbol(856 B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,857 jitlink::Scope::SideEffectsOnly, false, true);858 }859 860 // Add keep-alive edges to anonymous symbols in all other init blocks.861 for (auto *B : InitSection.blocks()) {862 if (B == &InitSym->getBlock())863 continue;864 865 auto &S = G.addAnonymousSymbol(*B, 0, B->getSize(), false, true);866 InitSym->getBlock().addEdge(jitlink::Edge::KeepAlive, 0, S, 0);867 }868 }869 }870 871 return Error::success();872}873 874Error COFFPlatform::COFFPlatformPlugin::875 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,876 JITDylib &JD) {877 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);878 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];879 COFFObjectSectionsMap ObjSecs;880 for (auto &S : G.sections()) {881 jitlink::SectionRange Range(S);882 if (Range.getSize())883 ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));884 }885 886 G.allocActions().push_back(887 {{},888 cantFail(889 WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(890 CP.orc_rt_coff_deregister_object_sections, HeaderAddr,891 ObjSecs))});892 893 auto &BState = CP.JDBootstrapStates[&JD];894 BState.ObjectSectionsMaps.push_back(std::move(ObjSecs));895 896 // Collect static initializers897 for (auto &S : G.sections())898 if (isCOFFInitializerSection(S.getName()))899 for (auto *B : S.blocks()) {900 if (B->edges_empty())901 continue;902 for (auto &E : B->edges())903 BState.Initializers.push_back(std::make_pair(904 S.getName().str(), E.getTarget().getAddress() + E.getAddend()));905 }906 907 return Error::success();908}909 910} // End namespace orc.911} // End namespace llvm.912