1565 lines · cpp
1//===- macho_platform.cpp -------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains code required to load the rest of the MachO runtime.10//11//===----------------------------------------------------------------------===//12 13#include "macho_platform.h"14#include "bitmask_enum.h"15#include "common.h"16#include "debug.h"17#include "error.h"18#include "interval_map.h"19#include "jit_dispatch.h"20#include "record_section_tracker.h"21#include "wrapper_function_utils.h"22 23#include <algorithm>24#include <ios>25#include <map>26#include <mutex>27#include <sstream>28#include <string_view>29#include <unordered_map>30#include <unordered_set>31#include <vector>32 33#define DEBUG_TYPE "macho_platform"34 35using namespace orc_rt;36using namespace orc_rt::macho;37 38// Declare function tags for functions in the JIT process.39ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag)40ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_symbols_tag)41 42struct objc_image_info;43struct mach_header;44 45// Objective-C registration functions.46// These are weakly imported. If the Objective-C runtime has not been loaded47// then code containing Objective-C sections will generate an error.48extern "C" void49_objc_map_images(unsigned count, const char *const paths[],50 const mach_header *const mhdrs[]) ORC_RT_WEAK_IMPORT;51 52extern "C" void _objc_load_image(const char *path,53 const mach_header *mh) ORC_RT_WEAK_IMPORT;54 55// Libunwind prototypes.56struct unw_dynamic_unwind_sections {57 uintptr_t dso_base;58 uintptr_t dwarf_section;59 size_t dwarf_section_length;60 uintptr_t compact_unwind_section;61 size_t compact_unwind_section_length;62};63 64typedef int (*unw_find_dynamic_unwind_sections)(65 uintptr_t addr, struct unw_dynamic_unwind_sections *info);66 67extern "C" int __unw_add_find_dynamic_unwind_sections(68 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)69 ORC_RT_WEAK_IMPORT;70 71extern "C" int __unw_remove_find_dynamic_unwind_sections(72 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)73 ORC_RT_WEAK_IMPORT;74 75namespace {76 77struct MachOJITDylibDepInfo {78 bool Sealed = false;79 std::vector<ExecutorAddr> DepHeaders;80};81 82using MachOJITDylibDepInfoMap =83 std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>;84 85} // anonymous namespace86 87namespace orc_rt {88 89using SPSMachOObjectPlatformSectionsMap =90 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;91 92using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;93 94using SPSMachOJITDylibDepInfoMap =95 SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;96 97template <>98class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> {99public:100 static size_t size(const MachOJITDylibDepInfo &JDI) {101 return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders);102 }103 104 static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) {105 return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed,106 JDI.DepHeaders);107 }108 109 static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) {110 return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed,111 JDI.DepHeaders);112 }113};114 115struct UnwindSectionInfo {116 std::vector<ExecutorAddrRange> CodeRanges;117 ExecutorAddrRange DwarfSection;118 ExecutorAddrRange CompactUnwindSection;119};120 121using SPSUnwindSectionInfo =122 SPSTuple<SPSSequence<SPSExecutorAddrRange>, SPSExecutorAddrRange,123 SPSExecutorAddrRange>;124 125template <>126class SPSSerializationTraits<SPSUnwindSectionInfo, UnwindSectionInfo> {127public:128 static size_t size(const UnwindSectionInfo &USI) {129 return SPSUnwindSectionInfo::AsArgList::size(130 USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);131 }132 133 static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) {134 return SPSUnwindSectionInfo::AsArgList::serialize(135 OB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);136 }137 138 static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) {139 return SPSUnwindSectionInfo::AsArgList::deserialize(140 IB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);141 }142};143 144} // namespace orc_rt145 146namespace {147struct TLVDescriptor {148 void *(*Thunk)(TLVDescriptor *) = nullptr;149 unsigned long Key = 0;150 unsigned long DataAddress = 0;151};152 153class MachOPlatformRuntimeState {154public:155 // Used internally by MachOPlatformRuntimeState, but made public to enable156 // serialization.157 enum class MachOExecutorSymbolFlags : uint8_t {158 None = 0,159 Weak = 1U << 0,160 Callable = 1U << 1,161 ORC_RT_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable)162 };163 164private:165 struct AtExitEntry {166 void (*Func)(void *);167 void *Arg;168 };169 170 using AtExitsVector = std::vector<AtExitEntry>;171 172 struct UnwindSections {173 UnwindSections(const UnwindSectionInfo &USI)174 : DwarfSection(USI.DwarfSection.toSpan<char>()),175 CompactUnwindSection(USI.CompactUnwindSection.toSpan<char>()) {}176 177 span<char> DwarfSection;178 span<char> CompactUnwindSection;179 };180 181 using UnwindSectionsMap =182 IntervalMap<char *, UnwindSections, IntervalCoalescing::Disabled>;183 184 struct JITDylibState {185 186 using SymbolTableMap =187 std::unordered_map<std::string_view,188 std::pair<ExecutorAddr, MachOExecutorSymbolFlags>>;189 190 std::string Name;191 void *Header = nullptr;192 bool Sealed = false;193 size_t LinkedAgainstRefCount = 0;194 size_t DlRefCount = 0;195 SymbolTableMap SymbolTable;196 std::vector<JITDylibState *> Deps;197 AtExitsVector AtExits;198 const objc_image_info *ObjCImageInfo = nullptr;199 std::unordered_map<void *, std::vector<char>> DataSectionContent;200 std::unordered_map<void *, size_t> ZeroInitRanges;201 UnwindSectionsMap UnwindSections;202 RecordSectionsTracker<void (*)()> ModInitsSections;203 RecordSectionsTracker<char> ObjCRuntimeRegistrationObjects;204 205 bool referenced() const {206 return LinkedAgainstRefCount != 0 || DlRefCount != 0;207 }208 };209 210public:211 static Error create();212 static MachOPlatformRuntimeState &get();213 static Error destroy();214 215 MachOPlatformRuntimeState() = default;216 217 // Delete copy and move constructors.218 MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete;219 MachOPlatformRuntimeState &220 operator=(const MachOPlatformRuntimeState &) = delete;221 MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete;222 MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete;223 224 Error initialize();225 Error shutdown();226 227 Error registerJITDylib(std::string Name, void *Header);228 Error deregisterJITDylib(void *Header);229 Error registerThreadDataSection(span<const char> ThreadDataSection);230 Error deregisterThreadDataSection(span<const char> ThreadDataSection);231 Error registerObjectSymbolTable(232 ExecutorAddr HeaderAddr,233 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,234 MachOExecutorSymbolFlags>> &Entries);235 Error deregisterObjectSymbolTable(236 ExecutorAddr HeaderAddr,237 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,238 MachOExecutorSymbolFlags>> &Entries);239 Error registerObjectPlatformSections(240 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,241 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);242 Error deregisterObjectPlatformSections(243 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,244 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);245 246 const char *dlerror();247 void *dlopen(std::string_view Name, int Mode);248 int dlupdate(void *DSOHandle);249 int dlclose(void *DSOHandle);250 void *dlsym(void *DSOHandle, const char *Symbol);251 252 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);253 void runAtExits(std::unique_lock<std::mutex> &JDStatesLock,254 JITDylibState &JDS);255 void runAtExits(void *DSOHandle);256 257 /// Returns the base address of the section containing ThreadData.258 Expected<std::pair<const char *, size_t>>259 getThreadDataSectionFor(const char *ThreadData);260 261private:262 JITDylibState *getJITDylibStateByHeader(void *DSOHandle);263 JITDylibState *getJITDylibStateByName(std::string_view Path);264 265 /// Requests materialization of the given symbols. For each pair, the bool266 /// element indicates whether the symbol is required (true) or weakly267 /// referenced (false).268 Error requestPushSymbols(JITDylibState &JDS,269 span<std::pair<std::string_view, bool>> Symbols);270 271 /// Attempts to look up the given symbols locally, requesting a push from the272 /// remote if they're not found. Results are written to the Result span, which273 /// must have the same size as the Symbols span.274 Error275 lookupSymbols(JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock,276 span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result,277 span<std::pair<std::string_view, bool>> Symbols);278 279 bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info);280 281 static int findDynamicUnwindSections(uintptr_t addr,282 unw_dynamic_unwind_sections *info);283 static Error registerEHFrames(span<const char> EHFrameSection);284 static Error deregisterEHFrames(span<const char> EHFrameSection);285 286 static Error287 registerObjCRegistrationObjects(std::unique_lock<std::mutex> &JDStatesLock,288 JITDylibState &JDS);289 static Error runModInits(std::unique_lock<std::mutex> &JDStatesLock,290 JITDylibState &JDS);291 292 Expected<void *> dlopenImpl(std::string_view Path, int Mode);293 Error dlopenFull(std::unique_lock<std::mutex> &JDStatesLock,294 JITDylibState &JDS);295 Error dlopenInitialize(std::unique_lock<std::mutex> &JDStatesLock,296 JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo);297 298 Error dlupdateImpl(void *DSOHandle);299 Error dlupdateFull(std::unique_lock<std::mutex> &JDStatesLock,300 JITDylibState &JDS);301 Error dlupdateInitialize(std::unique_lock<std::mutex> &JDStatesLock,302 JITDylibState &JDS);303 304 Error dlcloseImpl(void *DSOHandle);305 Error dlcloseDeinitialize(std::unique_lock<std::mutex> &JDStatesLock,306 JITDylibState &JDS);307 308 static MachOPlatformRuntimeState *MOPS;309 310 bool UseCallbackStyleUnwindInfo = false;311 312 // FIXME: Move to thread-state.313 std::string DLFcnError;314 315 // APIMutex guards against concurrent entry into key "dyld" API functions316 // (e.g. dlopen, dlclose).317 std::recursive_mutex DyldAPIMutex;318 319 // JDStatesMutex guards the data structures that hold JITDylib state.320 std::mutex JDStatesMutex;321 std::unordered_map<void *, JITDylibState> JDStates;322 std::unordered_map<std::string_view, void *> JDNameToHeader;323 324 // ThreadDataSectionsMutex guards thread local data section state.325 std::mutex ThreadDataSectionsMutex;326 std::map<const char *, size_t> ThreadDataSections;327};328 329} // anonymous namespace330 331namespace orc_rt {332 333class SPSMachOExecutorSymbolFlags;334 335template <>336class SPSSerializationTraits<337 SPSMachOExecutorSymbolFlags,338 MachOPlatformRuntimeState::MachOExecutorSymbolFlags> {339private:340 using UT = std::underlying_type_t<341 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>;342 343public:344 static size_t345 size(const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {346 return sizeof(UT);347 }348 349 static bool350 serialize(SPSOutputBuffer &OB,351 const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {352 return SPSArgList<UT>::serialize(OB, static_cast<UT>(SF));353 }354 355 static bool356 deserialize(SPSInputBuffer &IB,357 MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {358 UT Tmp;359 if (!SPSArgList<UT>::deserialize(IB, Tmp))360 return false;361 SF = static_cast<MachOPlatformRuntimeState::MachOExecutorSymbolFlags>(Tmp);362 return true;363 }364};365 366} // namespace orc_rt367 368namespace {369 370MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr;371 372Error MachOPlatformRuntimeState::create() {373 assert(!MOPS && "MachOPlatformRuntimeState should be null");374 MOPS = new MachOPlatformRuntimeState();375 return MOPS->initialize();376}377 378MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() {379 assert(MOPS && "MachOPlatformRuntimeState not initialized");380 return *MOPS;381}382 383Error MachOPlatformRuntimeState::destroy() {384 assert(MOPS && "MachOPlatformRuntimeState not initialized");385 auto Err = MOPS->shutdown();386 delete MOPS;387 return Err;388}389 390Error MachOPlatformRuntimeState::initialize() {391 UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections &&392 __unw_remove_find_dynamic_unwind_sections;393 if (UseCallbackStyleUnwindInfo) {394 ORC_RT_DEBUG({395 printdbg("__unw_add/remove_find_dynamic_unwind_sections available."396 " Using callback-based frame info lookup.\n");397 });398 if (__unw_add_find_dynamic_unwind_sections(&findDynamicUnwindSections))399 return make_error<StringError>(400 "Could not register findDynamicUnwindSections");401 } else {402 ORC_RT_DEBUG({403 printdbg("__unw_add/remove_find_dynamic_unwind_sections not available."404 " Using classic frame info registration.\n");405 });406 }407 return Error::success();408}409 410Error MachOPlatformRuntimeState::shutdown() {411 if (UseCallbackStyleUnwindInfo) {412 if (__unw_remove_find_dynamic_unwind_sections(&findDynamicUnwindSections)) {413 ORC_RT_DEBUG(414 { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); });415 }416 }417 return Error::success();418}419 420Error MachOPlatformRuntimeState::registerJITDylib(std::string Name,421 void *Header) {422 ORC_RT_DEBUG({423 printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header);424 });425 std::lock_guard<std::mutex> Lock(JDStatesMutex);426 if (JDStates.count(Header)) {427 std::ostringstream ErrStream;428 ErrStream << "Duplicate JITDylib registration for header " << Header429 << " (name = " << Name << ")";430 return make_error<StringError>(ErrStream.str());431 }432 if (JDNameToHeader.count(Name)) {433 std::ostringstream ErrStream;434 ErrStream << "Duplicate JITDylib registration for header " << Header435 << " (header = " << Header << ")";436 return make_error<StringError>(ErrStream.str());437 }438 439 auto &JDS = JDStates[Header];440 JDS.Name = std::move(Name);441 JDS.Header = Header;442 JDNameToHeader[JDS.Name] = Header;443 return Error::success();444}445 446Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) {447 std::lock_guard<std::mutex> Lock(JDStatesMutex);448 auto I = JDStates.find(Header);449 if (I == JDStates.end()) {450 std::ostringstream ErrStream;451 ErrStream << "Attempted to deregister unrecognized header " << Header;452 return make_error<StringError>(ErrStream.str());453 }454 455 // Remove std::string construction once we can use C++20.456 auto J = JDNameToHeader.find(457 std::string(I->second.Name.data(), I->second.Name.size()));458 assert(J != JDNameToHeader.end() &&459 "Missing JDNameToHeader entry for JITDylib");460 461 ORC_RT_DEBUG({462 printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(),463 Header);464 });465 466 JDNameToHeader.erase(J);467 JDStates.erase(I);468 return Error::success();469}470 471Error MachOPlatformRuntimeState::registerThreadDataSection(472 span<const char> ThreadDataSection) {473 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);474 auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());475 if (I != ThreadDataSections.begin()) {476 auto J = std::prev(I);477 if (J->first + J->second > ThreadDataSection.data())478 return make_error<StringError>("Overlapping __thread_data sections");479 }480 ThreadDataSections.insert(481 I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));482 return Error::success();483}484 485Error MachOPlatformRuntimeState::deregisterThreadDataSection(486 span<const char> ThreadDataSection) {487 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);488 auto I = ThreadDataSections.find(ThreadDataSection.data());489 if (I == ThreadDataSections.end())490 return make_error<StringError>("Attempt to deregister unknown thread data "491 "section");492 ThreadDataSections.erase(I);493 return Error::success();494}495 496Error MachOPlatformRuntimeState::registerObjectSymbolTable(497 ExecutorAddr HeaderAddr,498 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,499 MachOExecutorSymbolFlags>> &Entries) {500 501 std::lock_guard<std::mutex> Lock(JDStatesMutex);502 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());503 if (!JDS) {504 std::ostringstream ErrStream;505 ErrStream << "Could not register object platform sections for "506 "unrecognized header "507 << HeaderAddr.toPtr<void *>();508 return make_error<StringError>(ErrStream.str());509 }510 511 for (auto &[NameAddr, SymAddr, Flags] : Entries)512 JDS->SymbolTable[NameAddr.toPtr<const char *>()] = {SymAddr, Flags};513 514 return Error::success();515}516 517Error MachOPlatformRuntimeState::deregisterObjectSymbolTable(518 ExecutorAddr HeaderAddr,519 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,520 MachOExecutorSymbolFlags>> &Entries) {521 522 std::lock_guard<std::mutex> Lock(JDStatesMutex);523 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());524 if (!JDS) {525 std::ostringstream ErrStream;526 ErrStream << "Could not register object platform sections for "527 "unrecognized header "528 << HeaderAddr.toPtr<void *>();529 return make_error<StringError>(ErrStream.str());530 }531 532 for (auto &[NameAddr, SymAddr, Flags] : Entries)533 JDS->SymbolTable.erase(NameAddr.toPtr<const char *>());534 535 return Error::success();536}537 538Error MachOPlatformRuntimeState::registerObjectPlatformSections(539 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,540 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {541 542 // FIXME: Reject platform section registration after the JITDylib is543 // sealed?544 545 ORC_RT_DEBUG({546 printdbg("MachOPlatform: Registering object sections for %p.\n",547 HeaderAddr.toPtr<void *>());548 });549 550 std::lock_guard<std::mutex> Lock(JDStatesMutex);551 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());552 if (!JDS) {553 std::ostringstream ErrStream;554 ErrStream << "Could not register object platform sections for "555 "unrecognized header "556 << HeaderAddr.toPtr<void *>();557 return make_error<StringError>(ErrStream.str());558 }559 560 ORC_RT_DEBUG({561 printdbg(" UnwindInfo: %s, UseCallbackStyleUnwindInfo: %s\n",562 UnwindInfo ? "true" : "false",563 UseCallbackStyleUnwindInfo ? "true" : "false");564 });565 566 if (UnwindInfo && UseCallbackStyleUnwindInfo) {567 ORC_RT_DEBUG({568 printdbg(" Registering new-style unwind info for:\n"569 " DWARF: %p -- %p\n"570 " Compact-unwind: %p -- %p\n"571 " for:\n",572 UnwindInfo->DwarfSection.Start.toPtr<void *>(),573 UnwindInfo->DwarfSection.End.toPtr<void *>(),574 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),575 UnwindInfo->CompactUnwindSection.End.toPtr<void *>());576 });577 for (auto &CodeRange : UnwindInfo->CodeRanges) {578 JDS->UnwindSections.insert(CodeRange.Start.toPtr<char *>(),579 CodeRange.End.toPtr<char *>(), *UnwindInfo);580 ORC_RT_DEBUG({581 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),582 CodeRange.End.toPtr<void *>());583 });584 }585 }586 587 for (auto &KV : Secs) {588 // FIXME: Validate section ranges?589 if (KV.first == "__TEXT,__eh_frame") {590 if (!UseCallbackStyleUnwindInfo) {591 // Use classic libunwind registration.592 if (auto Err = registerEHFrames(KV.second.toSpan<const char>()))593 return Err;594 }595 } else if (KV.first == "__DATA,__data") {596 assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) &&597 "Address already registered.");598 auto S = KV.second.toSpan<char>();599 JDS->DataSectionContent[KV.second.Start.toPtr<char *>()] =600 std::vector<char>(S.begin(), S.end());601 } else if (KV.first == "__DATA,__common") {602 JDS->ZeroInitRanges[KV.second.Start.toPtr<char *>()] = KV.second.size();603 } else if (KV.first == "__DATA,__thread_data") {604 if (auto Err = registerThreadDataSection(KV.second.toSpan<const char>()))605 return Err;606 } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")607 JDS->ObjCRuntimeRegistrationObjects.add(KV.second.toSpan<char>());608 else if (KV.first == "__DATA,__mod_init_func")609 JDS->ModInitsSections.add(KV.second.toSpan<void (*)()>());610 else {611 // Should this be a warning instead?612 return make_error<StringError>(613 "Encountered unexpected section " +614 std::string(KV.first.data(), KV.first.size()) +615 " while registering object platform sections");616 }617 }618 619 return Error::success();620}621 622Error MachOPlatformRuntimeState::deregisterObjectPlatformSections(623 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,624 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {625 // TODO: Make this more efficient? (maybe unnecessary if removal is rare?)626 // TODO: Add a JITDylib prepare-for-teardown operation that clears all627 // registered sections, causing this function to take the fast-path.628 ORC_RT_DEBUG({629 printdbg("MachOPlatform: Deregistering object sections for %p.\n",630 HeaderAddr.toPtr<void *>());631 });632 633 std::lock_guard<std::mutex> Lock(JDStatesMutex);634 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());635 if (!JDS) {636 std::ostringstream ErrStream;637 ErrStream << "Could not register object platform sections for unrecognized "638 "header "639 << HeaderAddr.toPtr<void *>();640 return make_error<StringError>(ErrStream.str());641 }642 643 // FIXME: Implement faster-path by returning immediately if JDS is being644 // torn down entirely?645 646 // TODO: Make library permanent (i.e. not able to be dlclosed) if it contains647 // any Swift or ObjC. Once this happens we can clear (and no longer record)648 // data section content, as the library could never be re-initialized.649 650 if (UnwindInfo && UseCallbackStyleUnwindInfo) {651 ORC_RT_DEBUG({652 printdbg(" Deregistering new-style unwind info for:\n"653 " DWARF: %p -- %p\n"654 " Compact-unwind: %p -- %p\n"655 " for:\n",656 UnwindInfo->DwarfSection.Start.toPtr<void *>(),657 UnwindInfo->DwarfSection.End.toPtr<void *>(),658 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),659 UnwindInfo->CompactUnwindSection.End.toPtr<void *>());660 });661 for (auto &CodeRange : UnwindInfo->CodeRanges) {662 JDS->UnwindSections.erase(CodeRange.Start.toPtr<char *>(),663 CodeRange.End.toPtr<char *>());664 ORC_RT_DEBUG({665 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),666 CodeRange.End.toPtr<void *>());667 });668 }669 }670 671 for (auto &KV : Secs) {672 // FIXME: Validate section ranges?673 if (KV.first == "__TEXT,__eh_frame") {674 if (!UseCallbackStyleUnwindInfo) {675 // Use classic libunwind registration.676 if (auto Err = deregisterEHFrames(KV.second.toSpan<const char>()))677 return Err;678 }679 } else if (KV.first == "__DATA,__data") {680 JDS->DataSectionContent.erase(KV.second.Start.toPtr<char *>());681 } else if (KV.first == "__DATA,__common") {682 JDS->ZeroInitRanges.erase(KV.second.Start.toPtr<char *>());683 } else if (KV.first == "__DATA,__thread_data") {684 if (auto Err =685 deregisterThreadDataSection(KV.second.toSpan<const char>()))686 return Err;687 } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")688 JDS->ObjCRuntimeRegistrationObjects.removeIfPresent(KV.second);689 else if (KV.first == "__DATA,__mod_init_func")690 JDS->ModInitsSections.removeIfPresent(KV.second);691 else {692 // Should this be a warning instead?693 return make_error<StringError>(694 "Encountered unexpected section " +695 std::string(KV.first.data(), KV.first.size()) +696 " while deregistering object platform sections");697 }698 }699 return Error::success();700}701 702const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }703 704void *MachOPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {705 ORC_RT_DEBUG({706 std::string S(Path.data(), Path.size());707 printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str());708 });709 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);710 if (auto H = dlopenImpl(Path, Mode))711 return *H;712 else {713 // FIXME: Make dlerror thread safe.714 DLFcnError = toString(H.takeError());715 return nullptr;716 }717}718 719int MachOPlatformRuntimeState::dlupdate(void *DSOHandle) {720 ORC_RT_DEBUG({721 std::string S;722 printdbg("MachOPlatform::dlupdate(%p) (%s)\n", DSOHandle, S.c_str());723 });724 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);725 if (auto Err = dlupdateImpl(DSOHandle)) {726 // FIXME: Make dlerror thread safe.727 DLFcnError = toString(std::move(Err));728 return -1;729 }730 return 0;731}732 733int MachOPlatformRuntimeState::dlclose(void *DSOHandle) {734 ORC_RT_DEBUG({735 auto *JDS = getJITDylibStateByHeader(DSOHandle);736 std::string DylibName;737 if (JDS) {738 std::string S;739 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str());740 } else741 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle,742 "invalid handle");743 });744 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);745 if (auto Err = dlcloseImpl(DSOHandle)) {746 // FIXME: Make dlerror thread safe.747 DLFcnError = toString(std::move(Err));748 return -1;749 }750 return 0;751}752 753void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, const char *Symbol) {754 std::unique_lock<std::mutex> Lock(JDStatesMutex);755 auto *JDS = getJITDylibStateByHeader(DSOHandle);756 if (!JDS) {757 std::ostringstream ErrStream;758 ErrStream << "In call to dlsym, unrecognized header address " << DSOHandle;759 DLFcnError = ErrStream.str();760 return nullptr;761 }762 763 std::string MangledName = std::string("_") + Symbol;764 std::pair<std::string_view, bool> Lookup(MangledName, false);765 std::pair<ExecutorAddr, MachOExecutorSymbolFlags> Result;766 767 if (auto Err = lookupSymbols(*JDS, Lock, {&Result, 1}, {&Lookup, 1})) {768 DLFcnError = toString(std::move(Err));769 return nullptr;770 }771 772 // Sign callable symbols as functions, to match dyld.773 if ((Result.second & MachOExecutorSymbolFlags::Callable) ==774 MachOExecutorSymbolFlags::Callable)775 return reinterpret_cast<void *>(Result.first.toPtr<void(void)>());776 return Result.first.toPtr<void *>();777}778 779int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,780 void *DSOHandle) {781 // FIXME: Handle out-of-memory errors, returning -1 if OOM.782 std::lock_guard<std::mutex> Lock(JDStatesMutex);783 auto *JDS = getJITDylibStateByHeader(DSOHandle);784 if (!JDS) {785 ORC_RT_DEBUG({786 printdbg("MachOPlatformRuntimeState::registerAtExit called with "787 "unrecognized dso handle %p\n",788 DSOHandle);789 });790 return -1;791 }792 JDS->AtExits.push_back({F, Arg});793 return 0;794}795 796void MachOPlatformRuntimeState::runAtExits(797 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {798 auto AtExits = std::move(JDS.AtExits);799 800 // Unlock while running atexits, as they may trigger operations that modify801 // JDStates.802 JDStatesLock.unlock();803 while (!AtExits.empty()) {804 auto &AE = AtExits.back();805 AE.Func(AE.Arg);806 AtExits.pop_back();807 }808 JDStatesLock.lock();809}810 811void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {812 std::unique_lock<std::mutex> Lock(JDStatesMutex);813 auto *JDS = getJITDylibStateByHeader(DSOHandle);814 ORC_RT_DEBUG({815 printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized "816 "dso_handle %p\n",817 DSOHandle);818 });819 if (JDS)820 runAtExits(Lock, *JDS);821}822 823Expected<std::pair<const char *, size_t>>824MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {825 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);826 auto I = ThreadDataSections.upper_bound(ThreadData);827 // Check that we have a valid entry covering this address.828 if (I == ThreadDataSections.begin())829 return make_error<StringError>("No thread local data section for key");830 I = std::prev(I);831 if (ThreadData >= I->first + I->second)832 return make_error<StringError>("No thread local data section for key");833 return *I;834}835 836MachOPlatformRuntimeState::JITDylibState *837MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) {838 auto I = JDStates.find(DSOHandle);839 if (I == JDStates.end()) {840 I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first;841 I->second.Header = DSOHandle;842 }843 return &I->second;844}845 846MachOPlatformRuntimeState::JITDylibState *847MachOPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {848 // FIXME: Avoid creating string once we have C++20.849 auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));850 if (I != JDNameToHeader.end())851 return getJITDylibStateByHeader(I->second);852 return nullptr;853}854 855Error MachOPlatformRuntimeState::requestPushSymbols(856 JITDylibState &JDS, span<std::pair<std::string_view, bool>> Symbols) {857 Error OpErr = Error::success();858 if (auto Err = WrapperFunction<SPSError(859 SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>)>::860 call(JITDispatch(&__orc_rt_macho_push_symbols_tag), OpErr,861 ExecutorAddr::fromPtr(JDS.Header), Symbols)) {862 cantFail(std::move(OpErr));863 return std::move(Err);864 }865 return OpErr;866}867 868Error MachOPlatformRuntimeState::lookupSymbols(869 JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock,870 span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result,871 span<std::pair<std::string_view, bool>> Symbols) {872 assert(JDStatesLock.owns_lock() &&873 "JDStatesLock should be locked at call-site");874 assert(Result.size() == Symbols.size() &&875 "Results and Symbols span sizes should match");876 877 // Make an initial pass over the local symbol table.878 std::vector<size_t> MissingSymbolIndexes;879 for (size_t Idx = 0; Idx != Symbols.size(); ++Idx) {880 auto I = JDS.SymbolTable.find(Symbols[Idx].first);881 if (I != JDS.SymbolTable.end())882 Result[Idx] = I->second;883 else884 MissingSymbolIndexes.push_back(Idx);885 }886 887 // If everything has been resolved already then bail out early.888 if (MissingSymbolIndexes.empty())889 return Error::success();890 891 // Otherwise call back to the controller to try to request that the symbol892 // be materialized.893 std::vector<std::pair<std::string_view, bool>> MissingSymbols;894 MissingSymbols.reserve(MissingSymbolIndexes.size());895 ORC_RT_DEBUG({896 printdbg("requesting push of %i missing symbols...\n",897 MissingSymbolIndexes.size());898 });899 for (auto MissingIdx : MissingSymbolIndexes)900 MissingSymbols.push_back(Symbols[MissingIdx]);901 902 JDStatesLock.unlock();903 if (auto Err = requestPushSymbols(904 JDS, {MissingSymbols.data(), MissingSymbols.size()}))905 return Err;906 JDStatesLock.lock();907 908 // Try to resolve the previously missing symbols locally.909 std::vector<size_t> MissingRequiredSymbols;910 for (auto MissingIdx : MissingSymbolIndexes) {911 auto I = JDS.SymbolTable.find(Symbols[MissingIdx].first);912 if (I != JDS.SymbolTable.end())913 Result[MissingIdx] = I->second;914 else {915 if (Symbols[MissingIdx].second)916 MissingRequiredSymbols.push_back(MissingIdx);917 else918 Result[MissingIdx] = {ExecutorAddr(), {}};919 }920 }921 922 // Error out if any missing symbols could not be resolved.923 if (!MissingRequiredSymbols.empty()) {924 std::ostringstream ErrStream;925 ErrStream << "Lookup could not find required symbols: [ ";926 for (auto MissingIdx : MissingRequiredSymbols)927 ErrStream << "\"" << Symbols[MissingIdx].first << "\" ";928 ErrStream << "]";929 return make_error<StringError>(ErrStream.str());930 }931 932 return Error::success();933}934 935// eh-frame registration functions.936// We expect these to be available for all processes.937extern "C" void __register_frame(const void *);938extern "C" void __deregister_frame(const void *);939 940template <typename HandleFDEFn>941void walkEHFrameSection(span<const char> EHFrameSection,942 HandleFDEFn HandleFDE) {943 const char *CurCFIRecord = EHFrameSection.data();944 uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);945 946 while (CurCFIRecord != EHFrameSection.end() && Size != 0) {947 const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);948 if (Size == 0xffffffff)949 Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;950 else951 Size += 4;952 uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);953 954 if (Offset != 0)955 HandleFDE(CurCFIRecord);956 957 CurCFIRecord += Size;958 Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);959 }960}961 962bool MachOPlatformRuntimeState::lookupUnwindSections(963 void *Addr, unw_dynamic_unwind_sections &Info) {964 ORC_RT_DEBUG(965 { printdbg("Tried to lookup unwind-info via new lookup call.\n"); });966 std::lock_guard<std::mutex> Lock(JDStatesMutex);967 for (auto &KV : JDStates) {968 auto &JD = KV.second;969 auto I = JD.UnwindSections.find(reinterpret_cast<char *>(Addr));970 if (I != JD.UnwindSections.end()) {971 Info.dso_base = reinterpret_cast<uintptr_t>(JD.Header);972 Info.dwarf_section =973 reinterpret_cast<uintptr_t>(I->second.DwarfSection.data());974 Info.dwarf_section_length = I->second.DwarfSection.size();975 Info.compact_unwind_section =976 reinterpret_cast<uintptr_t>(I->second.CompactUnwindSection.data());977 Info.compact_unwind_section_length =978 I->second.CompactUnwindSection.size();979 return true;980 }981 }982 return false;983}984 985int MachOPlatformRuntimeState::findDynamicUnwindSections(986 uintptr_t addr, unw_dynamic_unwind_sections *info) {987 if (!info)988 return 0;989 return MachOPlatformRuntimeState::get().lookupUnwindSections((void *)addr,990 *info);991}992 993Error MachOPlatformRuntimeState::registerEHFrames(994 span<const char> EHFrameSection) {995 walkEHFrameSection(EHFrameSection, __register_frame);996 return Error::success();997}998 999Error MachOPlatformRuntimeState::deregisterEHFrames(1000 span<const char> EHFrameSection) {1001 walkEHFrameSection(EHFrameSection, __deregister_frame);1002 return Error::success();1003}1004 1005Error MachOPlatformRuntimeState::registerObjCRegistrationObjects(1006 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {1007 ORC_RT_DEBUG(printdbg("Registering Objective-C / Swift metadata.\n"));1008 1009 std::vector<char *> RegObjBases;1010 JDS.ObjCRuntimeRegistrationObjects.processNewSections(1011 [&](span<char> RegObj) { RegObjBases.push_back(RegObj.data()); });1012 1013 if (RegObjBases.empty())1014 return Error::success();1015 1016 if (!_objc_map_images || !_objc_load_image)1017 return make_error<StringError>(1018 "Could not register Objective-C / Swift metadata: _objc_map_images / "1019 "_objc_load_image not found");1020 1021 // Release the lock while calling out to libobjc in case +load methods cause1022 // reentering the orc runtime.1023 JDStatesLock.unlock();1024 std::vector<char *> Paths;1025 Paths.resize(RegObjBases.size());1026 _objc_map_images(RegObjBases.size(), Paths.data(),1027 reinterpret_cast<mach_header **>(RegObjBases.data()));1028 1029 for (void *RegObjBase : RegObjBases)1030 _objc_load_image(nullptr, reinterpret_cast<mach_header *>(RegObjBase));1031 JDStatesLock.lock();1032 1033 return Error::success();1034}1035 1036Error MachOPlatformRuntimeState::runModInits(1037 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {1038 std::vector<span<void (*)()>> InitSections;1039 InitSections.reserve(JDS.ModInitsSections.numNewSections());1040 1041 // Copy initializer sections: If the JITDylib is unsealed then the1042 // initializers could reach back into the JIT and cause more initializers to1043 // be added.1044 // FIXME: Skip unlock and run in-place on sealed JITDylibs?1045 JDS.ModInitsSections.processNewSections(1046 [&](span<void (*)()> Inits) { InitSections.push_back(Inits); });1047 1048 JDStatesLock.unlock();1049 for (auto InitSec : InitSections)1050 for (auto *Init : InitSec)1051 Init();1052 JDStatesLock.lock();1053 1054 return Error::success();1055}1056 1057Expected<void *> MachOPlatformRuntimeState::dlopenImpl(std::string_view Path,1058 int Mode) {1059 std::unique_lock<std::mutex> Lock(JDStatesMutex);1060 1061 // Try to find JITDylib state by name.1062 auto *JDS = getJITDylibStateByName(Path);1063 1064 if (!JDS)1065 return make_error<StringError>("No registered JTIDylib for path " +1066 std::string(Path.data(), Path.size()));1067 1068 // If this JITDylib is unsealed, or this is the first dlopen then run1069 // full dlopen path (update deps, push and run initializers, update ref1070 // counts on all JITDylibs in the dep tree).1071 if (!JDS->referenced() || !JDS->Sealed) {1072 if (auto Err = dlopenFull(Lock, *JDS))1073 return std::move(Err);1074 }1075 1076 // Bump the ref-count on this dylib.1077 ++JDS->DlRefCount;1078 1079 // Return the header address.1080 return JDS->Header;1081}1082 1083Error MachOPlatformRuntimeState::dlopenFull(1084 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {1085 // Call back to the JIT to push the initializers.1086 Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));1087 // Unlock so that we can accept the initializer update.1088 JDStatesLock.unlock();1089 if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(1090 SPSExecutorAddr)>::1091 call(JITDispatch(&__orc_rt_macho_push_initializers_tag), DepInfo,1092 ExecutorAddr::fromPtr(JDS.Header)))1093 return Err;1094 JDStatesLock.lock();1095 1096 if (!DepInfo)1097 return DepInfo.takeError();1098 1099 if (auto Err = dlopenInitialize(JDStatesLock, JDS, *DepInfo))1100 return Err;1101 1102 if (!DepInfo->empty()) {1103 ORC_RT_DEBUG({1104 printdbg("Unrecognized dep-info key headers in dlopen of %s\n",1105 JDS.Name.c_str());1106 });1107 std::ostringstream ErrStream;1108 ErrStream << "Encountered unrecognized dep-info key headers "1109 "while processing dlopen of "1110 << JDS.Name;1111 return make_error<StringError>(ErrStream.str());1112 }1113 1114 return Error::success();1115}1116 1117Error MachOPlatformRuntimeState::dlopenInitialize(1118 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS,1119 MachOJITDylibDepInfoMap &DepInfo) {1120 ORC_RT_DEBUG({1121 printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n",1122 JDS.Name.c_str());1123 });1124 1125 // If the header is not present in the dep map then assume that we1126 // already processed it earlier in the dlopenInitialize traversal and1127 // return.1128 // TODO: Keep a visited set instead so that we can error out on missing1129 // entries?1130 auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));1131 if (I == DepInfo.end())1132 return Error::success();1133 1134 auto DI = std::move(I->second);1135 DepInfo.erase(I);1136 1137 // We don't need to re-initialize sealed JITDylibs that have already been1138 // initialized. Just check that their dep-map entry is empty as expected.1139 if (JDS.Sealed) {1140 if (!DI.DepHeaders.empty()) {1141 std::ostringstream ErrStream;1142 ErrStream << "Sealed JITDylib " << JDS.Header1143 << " already has registered dependencies";1144 return make_error<StringError>(ErrStream.str());1145 }1146 if (JDS.referenced())1147 return Error::success();1148 } else1149 JDS.Sealed = DI.Sealed;1150 1151 // This is an unsealed or newly sealed JITDylib. Run initializers.1152 std::vector<JITDylibState *> OldDeps;1153 std::swap(JDS.Deps, OldDeps);1154 JDS.Deps.reserve(DI.DepHeaders.size());1155 for (auto DepHeaderAddr : DI.DepHeaders) {1156 auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>());1157 if (!DepJDS) {1158 std::ostringstream ErrStream;1159 ErrStream << "Encountered unrecognized dep header "1160 << DepHeaderAddr.toPtr<void *>() << " while initializing "1161 << JDS.Name;1162 return make_error<StringError>(ErrStream.str());1163 }1164 ++DepJDS->LinkedAgainstRefCount;1165 if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo))1166 return Err;1167 }1168 1169 // Initialize this JITDylib.1170 if (auto Err = registerObjCRegistrationObjects(JDStatesLock, JDS))1171 return Err;1172 if (auto Err = runModInits(JDStatesLock, JDS))1173 return Err;1174 1175 // Decrement old deps.1176 // FIXME: We should probably continue and just report deinitialize errors1177 // here.1178 for (auto *DepJDS : OldDeps) {1179 --DepJDS->LinkedAgainstRefCount;1180 if (!DepJDS->referenced())1181 if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))1182 return Err;1183 }1184 1185 return Error::success();1186}1187 1188Error MachOPlatformRuntimeState::dlupdateImpl(void *DSOHandle) {1189 std::unique_lock<std::mutex> Lock(JDStatesMutex);1190 1191 // Try to find JITDylib state by DSOHandle.1192 auto *JDS = getJITDylibStateByHeader(DSOHandle);1193 1194 if (!JDS) {1195 std::ostringstream ErrStream;1196 ErrStream << "No registered JITDylib for " << DSOHandle;1197 return make_error<StringError>(ErrStream.str());1198 }1199 1200 if (!JDS->referenced())1201 return make_error<StringError>("dlupdate failed, JITDylib must be open.");1202 1203 if (!JDS->Sealed) {1204 if (auto Err = dlupdateFull(Lock, *JDS))1205 return Err;1206 }1207 1208 return Error::success();1209}1210 1211Error MachOPlatformRuntimeState::dlupdateFull(1212 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {1213 // Call back to the JIT to push the initializers.1214 Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));1215 // Unlock so that we can accept the initializer update.1216 JDStatesLock.unlock();1217 if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(1218 SPSExecutorAddr)>::1219 call(JITDispatch(&__orc_rt_macho_push_initializers_tag), DepInfo,1220 ExecutorAddr::fromPtr(JDS.Header)))1221 return Err;1222 JDStatesLock.lock();1223 1224 if (!DepInfo)1225 return DepInfo.takeError();1226 1227 if (auto Err = dlupdateInitialize(JDStatesLock, JDS))1228 return Err;1229 1230 return Error::success();1231}1232 1233Error MachOPlatformRuntimeState::dlupdateInitialize(1234 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {1235 ORC_RT_DEBUG({1236 printdbg("MachOPlatformRuntimeState::dlupdateInitialize(\"%s\")\n",1237 JDS.Name.c_str());1238 });1239 1240 // Initialize this JITDylib.1241 if (auto Err = registerObjCRegistrationObjects(JDStatesLock, JDS))1242 return Err;1243 if (auto Err = runModInits(JDStatesLock, JDS))1244 return Err;1245 1246 return Error::success();1247}1248 1249Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {1250 std::unique_lock<std::mutex> Lock(JDStatesMutex);1251 1252 // Try to find JITDylib state by header.1253 auto *JDS = getJITDylibStateByHeader(DSOHandle);1254 1255 if (!JDS) {1256 std::ostringstream ErrStream;1257 ErrStream << "No registered JITDylib for " << DSOHandle;1258 return make_error<StringError>(ErrStream.str());1259 }1260 1261 // Bump the ref-count.1262 --JDS->DlRefCount;1263 1264 if (!JDS->referenced())1265 return dlcloseDeinitialize(Lock, *JDS);1266 1267 return Error::success();1268}1269 1270Error MachOPlatformRuntimeState::dlcloseDeinitialize(1271 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {1272 1273 ORC_RT_DEBUG({1274 printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",1275 JDS.Name.c_str());1276 });1277 1278 runAtExits(JDStatesLock, JDS);1279 1280 // Reset mod-inits1281 JDS.ModInitsSections.reset();1282 1283 // Reset data section contents.1284 for (auto &KV : JDS.DataSectionContent)1285 memcpy(KV.first, KV.second.data(), KV.second.size());1286 for (auto &KV : JDS.ZeroInitRanges)1287 memset(KV.first, 0, KV.second);1288 1289 // Deinitialize any dependencies.1290 for (auto *DepJDS : JDS.Deps) {1291 --DepJDS->LinkedAgainstRefCount;1292 if (!DepJDS->referenced())1293 if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))1294 return Err;1295 }1296 1297 return Error::success();1298}1299 1300class MachOPlatformRuntimeTLVManager {1301public:1302 void *getInstance(const char *ThreadData);1303 1304private:1305 std::unordered_map<const char *, char *> Instances;1306 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;1307};1308 1309void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {1310 auto I = Instances.find(ThreadData);1311 if (I != Instances.end())1312 return I->second;1313 1314 auto TDS =1315 MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);1316 if (!TDS) {1317 __orc_rt_log_error(toString(TDS.takeError()).c_str());1318 return nullptr;1319 }1320 1321 auto &Allocated = AllocatedSections[TDS->first];1322 if (!Allocated) {1323 Allocated = std::make_unique<char[]>(TDS->second);1324 memcpy(Allocated.get(), TDS->first, TDS->second);1325 }1326 1327 size_t ThreadDataDelta = ThreadData - TDS->first;1328 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");1329 1330 char *Instance = Allocated.get() + ThreadDataDelta;1331 Instances[ThreadData] = Instance;1332 return Instance;1333}1334 1335void destroyMachOTLVMgr(void *MachOTLVMgr) {1336 delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr);1337}1338 1339Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {1340 for (auto &WFC : WFCs)1341 if (auto Err = WFC.runWithSPSRet<void>())1342 return Err;1343 return Error::success();1344}1345 1346} // end anonymous namespace1347 1348//------------------------------------------------------------------------------1349// JIT entry points1350//------------------------------------------------------------------------------1351 1352ORC_RT_INTERFACE orc_rt_WrapperFunctionResult1353__orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {1354 return WrapperFunction<SPSError()>::handle(1355 ArgData, ArgSize,1356 []() { return MachOPlatformRuntimeState::create(); })1357 .release();1358}1359 1360ORC_RT_INTERFACE orc_rt_WrapperFunctionResult1361__orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {1362 return WrapperFunction<SPSError()>::handle(1363 ArgData, ArgSize,1364 []() { return MachOPlatformRuntimeState::destroy(); })1365 .release();1366}1367 1368ORC_RT_INTERFACE orc_rt_WrapperFunctionResult1369__orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {1370 return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(1371 ArgData, ArgSize,1372 [](std::string &Name, ExecutorAddr HeaderAddr) {1373 return MachOPlatformRuntimeState::get().registerJITDylib(1374 std::move(Name), HeaderAddr.toPtr<void *>());1375 })1376 .release();1377}1378 1379ORC_RT_INTERFACE orc_rt_WrapperFunctionResult1380__orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {1381 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(1382 ArgData, ArgSize,1383 [](ExecutorAddr HeaderAddr) {1384 return MachOPlatformRuntimeState::get().deregisterJITDylib(1385 HeaderAddr.toPtr<void *>());1386 })1387 .release();1388}1389 1390ORC_RT_INTERFACE orc_rt_WrapperFunctionResult1391__orc_rt_macho_register_object_platform_sections(char *ArgData,1392 size_t ArgSize) {1393 return WrapperFunction<SPSError(SPSExecutorAddr,1394 SPSOptional<SPSUnwindSectionInfo>,1395 SPSMachOObjectPlatformSectionsMap)>::1396 handle(ArgData, ArgSize,1397 [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,1398 std::vector<std::pair<std::string_view, ExecutorAddrRange>>1399 &Secs) {1400 return MachOPlatformRuntimeState::get()1401 .registerObjectPlatformSections(HeaderAddr, std::move(USI),1402 std::move(Secs));1403 })1404 .release();1405}1406 1407ORC_RT_INTERFACE orc_rt_WrapperFunctionResult1408__orc_rt_macho_register_object_symbol_table(char *ArgData, size_t ArgSize) {1409 using SymtabContainer = std::vector<1410 std::tuple<ExecutorAddr, ExecutorAddr,1411 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>;1412 return WrapperFunction<SPSError(1413 SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,1414 SPSMachOExecutorSymbolFlags>>)>::1415 handle(ArgData, ArgSize,1416 [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) {1417 return MachOPlatformRuntimeState::get()1418 .registerObjectSymbolTable(HeaderAddr, Symbols);1419 })1420 .release();1421}1422 1423ORC_RT_INTERFACE orc_rt_WrapperFunctionResult1424__orc_rt_macho_deregister_object_symbol_table(char *ArgData, size_t ArgSize) {1425 using SymtabContainer = std::vector<1426 std::tuple<ExecutorAddr, ExecutorAddr,1427 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>;1428 return WrapperFunction<SPSError(1429 SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,1430 SPSMachOExecutorSymbolFlags>>)>::1431 handle(ArgData, ArgSize,1432 [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) {1433 return MachOPlatformRuntimeState::get()1434 .deregisterObjectSymbolTable(HeaderAddr, Symbols);1435 })1436 .release();1437}1438 1439ORC_RT_INTERFACE orc_rt_WrapperFunctionResult1440__orc_rt_macho_deregister_object_platform_sections(char *ArgData,1441 size_t ArgSize) {1442 return WrapperFunction<SPSError(SPSExecutorAddr,1443 SPSOptional<SPSUnwindSectionInfo>,1444 SPSMachOObjectPlatformSectionsMap)>::1445 handle(ArgData, ArgSize,1446 [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,1447 std::vector<std::pair<std::string_view, ExecutorAddrRange>>1448 &Secs) {1449 return MachOPlatformRuntimeState::get()1450 .deregisterObjectPlatformSections(HeaderAddr, std::move(USI),1451 std::move(Secs));1452 })1453 .release();1454}1455 1456ORC_RT_INTERFACE orc_rt_WrapperFunctionResult1457__orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) {1458 return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle(1459 ArgData, ArgSize, runWrapperFunctionCalls)1460 .release();1461}1462 1463//------------------------------------------------------------------------------1464// TLV support1465//------------------------------------------------------------------------------1466 1467ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) {1468 auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>(1469 pthread_getspecific(D->Key));1470 if (!TLVMgr) {1471 TLVMgr = new MachOPlatformRuntimeTLVManager();1472 if (pthread_setspecific(D->Key, TLVMgr)) {1473 __orc_rt_log_error("Call to pthread_setspecific failed");1474 return nullptr;1475 }1476 }1477 1478 return TLVMgr->getInstance(1479 reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));1480}1481 1482ORC_RT_INTERFACE orc_rt_WrapperFunctionResult1483__orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) {1484 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(1485 ArgData, ArgSize,1486 []() -> Expected<uint64_t> {1487 pthread_key_t Key;1488 if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) {1489 __orc_rt_log_error("Call to pthread_key_create failed");1490 return make_error<StringError>(strerror(Err));1491 }1492 return static_cast<uint64_t>(Key);1493 })1494 .release();1495}1496 1497//------------------------------------------------------------------------------1498// cxa_atexit support1499//------------------------------------------------------------------------------1500 1501int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,1502 void *dso_handle) {1503 return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle);1504}1505 1506void __orc_rt_macho_cxa_finalize(void *dso_handle) {1507 MachOPlatformRuntimeState::get().runAtExits(dso_handle);1508}1509 1510//------------------------------------------------------------------------------1511// JIT'd dlfcn alternatives.1512//------------------------------------------------------------------------------1513 1514const char *__orc_rt_macho_jit_dlerror() {1515 return MachOPlatformRuntimeState::get().dlerror();1516}1517 1518void *__orc_rt_macho_jit_dlopen(const char *path, int mode) {1519 return MachOPlatformRuntimeState::get().dlopen(path, mode);1520}1521 1522int __orc_rt_macho_jit_dlupdate(void *dso_handle) {1523 return MachOPlatformRuntimeState::get().dlupdate(dso_handle);1524}1525 1526int __orc_rt_macho_jit_dlclose(void *dso_handle) {1527 return MachOPlatformRuntimeState::get().dlclose(dso_handle);1528}1529 1530void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) {1531 return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol);1532}1533 1534//------------------------------------------------------------------------------1535// MachO Run Program1536//------------------------------------------------------------------------------1537 1538ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName,1539 const char *EntrySymbolName,1540 int argc, char *argv[]) {1541 using MainTy = int (*)(int, char *[]);1542 1543 void *H =1544 __orc_rt_macho_jit_dlopen(JITDylibName, orc_rt::macho::ORC_RT_RTLD_LAZY);1545 if (!H) {1546 __orc_rt_log_error(__orc_rt_macho_jit_dlerror());1547 return -1;1548 }1549 1550 auto *Main =1551 reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName));1552 1553 if (!Main) {1554 __orc_rt_log_error(__orc_rt_macho_jit_dlerror());1555 return -1;1556 }1557 1558 int Result = Main(argc, argv);1559 1560 if (__orc_rt_macho_jit_dlclose(H) == -1)1561 __orc_rt_log_error(__orc_rt_macho_jit_dlerror());1562 1563 return Result;1564}1565