588 lines · cpp
1//===---- ExecutionUtils.cpp - Utilities for executing functions in Orc ---===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"10#include "llvm/ExecutionEngine/JITLink/x86_64.h"11#include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"12#include "llvm/ExecutionEngine/Orc/Layer.h"13#include "llvm/ExecutionEngine/Orc/LoadLinkableFile.h"14#include "llvm/ExecutionEngine/Orc/MachO.h"15#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"16#include "llvm/ExecutionEngine/Orc/SymbolStringPool.h"17#include "llvm/IR/Constants.h"18#include "llvm/IR/Function.h"19#include "llvm/IR/GlobalVariable.h"20#include "llvm/IR/Module.h"21#include "llvm/MC/TargetRegistry.h"22#include "llvm/Object/MachOUniversal.h"23#include "llvm/Target/TargetMachine.h"24#include <string>25 26namespace llvm {27namespace orc {28 29CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End)30 : InitList(31 GV ? dyn_cast_or_null<ConstantArray>(GV->getInitializer()) : nullptr),32 I((InitList && End) ? InitList->getNumOperands() : 0) {33}34 35bool CtorDtorIterator::operator==(const CtorDtorIterator &Other) const {36 assert(InitList == Other.InitList && "Incomparable iterators.");37 return I == Other.I;38}39 40bool CtorDtorIterator::operator!=(const CtorDtorIterator &Other) const {41 return !(*this == Other);42}43 44CtorDtorIterator& CtorDtorIterator::operator++() {45 ++I;46 return *this;47}48 49CtorDtorIterator CtorDtorIterator::operator++(int) {50 CtorDtorIterator Temp = *this;51 ++I;52 return Temp;53}54 55CtorDtorIterator::Element CtorDtorIterator::operator*() const {56 ConstantStruct *CS = dyn_cast<ConstantStruct>(InitList->getOperand(I));57 assert(CS && "Unrecognized type in llvm.global_ctors/llvm.global_dtors");58 59 Constant *FuncC = CS->getOperand(1);60 Function *Func = nullptr;61 62 // Extract function pointer, pulling off any casts.63 while (FuncC) {64 if (Function *F = dyn_cast_or_null<Function>(FuncC)) {65 Func = F;66 break;67 } else if (ConstantExpr *CE = dyn_cast_or_null<ConstantExpr>(FuncC)) {68 if (CE->isCast())69 FuncC = CE->getOperand(0);70 else71 break;72 } else {73 // This isn't anything we recognize. Bail out with Func left set to null.74 break;75 }76 }77 78 auto *Priority = cast<ConstantInt>(CS->getOperand(0));79 Value *Data = CS->getNumOperands() == 3 ? CS->getOperand(2) : nullptr;80 if (Data && !isa<GlobalValue>(Data))81 Data = nullptr;82 return Element(Priority->getZExtValue(), Func, Data);83}84 85iterator_range<CtorDtorIterator> getConstructors(const Module &M) {86 const GlobalVariable *CtorsList = M.getNamedGlobal("llvm.global_ctors");87 return make_range(CtorDtorIterator(CtorsList, false),88 CtorDtorIterator(CtorsList, true));89}90 91iterator_range<CtorDtorIterator> getDestructors(const Module &M) {92 const GlobalVariable *DtorsList = M.getNamedGlobal("llvm.global_dtors");93 return make_range(CtorDtorIterator(DtorsList, false),94 CtorDtorIterator(DtorsList, true));95}96 97bool StaticInitGVIterator::isStaticInitGlobal(GlobalValue &GV) {98 if (GV.isDeclaration())99 return false;100 101 if (GV.hasName() && (GV.getName() == "llvm.global_ctors" ||102 GV.getName() == "llvm.global_dtors"))103 return true;104 105 if (ObjFmt == Triple::MachO) {106 // FIXME: These section checks are too strict: We should match first and107 // second word split by comma.108 if (GV.hasSection() &&109 (GV.getSection().starts_with("__DATA,__objc_classlist") ||110 GV.getSection().starts_with("__DATA,__objc_selrefs")))111 return true;112 }113 114 return false;115}116 117void CtorDtorRunner::add(iterator_range<CtorDtorIterator> CtorDtors) {118 if (CtorDtors.empty())119 return;120 121 MangleAndInterner Mangle(122 JD.getExecutionSession(),123 (*CtorDtors.begin()).Func->getDataLayout());124 125 for (auto CtorDtor : CtorDtors) {126 assert(CtorDtor.Func && CtorDtor.Func->hasName() &&127 "Ctor/Dtor function must be named to be runnable under the JIT");128 129 // FIXME: Maybe use a symbol promoter here instead.130 if (CtorDtor.Func->hasLocalLinkage()) {131 CtorDtor.Func->setLinkage(GlobalValue::ExternalLinkage);132 CtorDtor.Func->setVisibility(GlobalValue::HiddenVisibility);133 }134 135 if (CtorDtor.Data && cast<GlobalValue>(CtorDtor.Data)->isDeclaration())136 continue;137 138 CtorDtorsByPriority[CtorDtor.Priority].push_back(139 Mangle(CtorDtor.Func->getName()));140 }141}142 143Error CtorDtorRunner::run() {144 using CtorDtorTy = void (*)();145 146 SymbolLookupSet LookupSet;147 for (auto &KV : CtorDtorsByPriority)148 for (auto &Name : KV.second)149 LookupSet.add(Name);150 assert(!LookupSet.containsDuplicates() &&151 "Ctor/Dtor list contains duplicates");152 153 auto &ES = JD.getExecutionSession();154 if (auto CtorDtorMap = ES.lookup(155 makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols),156 std::move(LookupSet))) {157 for (auto &KV : CtorDtorsByPriority) {158 for (auto &Name : KV.second) {159 assert(CtorDtorMap->count(Name) && "No entry for Name");160 auto CtorDtor = (*CtorDtorMap)[Name].getAddress().toPtr<CtorDtorTy>();161 CtorDtor();162 }163 }164 CtorDtorsByPriority.clear();165 return Error::success();166 } else167 return CtorDtorMap.takeError();168}169 170void LocalCXXRuntimeOverridesBase::runDestructors() {171 auto& CXXDestructorDataPairs = DSOHandleOverride;172 for (auto &P : CXXDestructorDataPairs)173 P.first(P.second);174 CXXDestructorDataPairs.clear();175}176 177int LocalCXXRuntimeOverridesBase::CXAAtExitOverride(DestructorPtr Destructor,178 void *Arg,179 void *DSOHandle) {180 auto& CXXDestructorDataPairs =181 *reinterpret_cast<CXXDestructorDataPairList*>(DSOHandle);182 CXXDestructorDataPairs.push_back(std::make_pair(Destructor, Arg));183 return 0;184}185 186Error LocalCXXRuntimeOverrides::enable(JITDylib &JD,187 MangleAndInterner &Mangle) {188 SymbolMap RuntimeInterposes;189 RuntimeInterposes[Mangle("__dso_handle")] = {190 ExecutorAddr::fromPtr(&DSOHandleOverride), JITSymbolFlags::Exported};191 RuntimeInterposes[Mangle("__cxa_atexit")] = {192 ExecutorAddr::fromPtr(&CXAAtExitOverride), JITSymbolFlags::Exported};193 194 return JD.define(absoluteSymbols(std::move(RuntimeInterposes)));195}196 197void ItaniumCXAAtExitSupport::registerAtExit(void (*F)(void *), void *Ctx,198 void *DSOHandle) {199 std::lock_guard<std::mutex> Lock(AtExitsMutex);200 AtExitRecords[DSOHandle].push_back({F, Ctx});201}202 203void ItaniumCXAAtExitSupport::runAtExits(void *DSOHandle) {204 std::vector<AtExitRecord> AtExitsToRun;205 206 {207 std::lock_guard<std::mutex> Lock(AtExitsMutex);208 auto I = AtExitRecords.find(DSOHandle);209 if (I != AtExitRecords.end()) {210 AtExitsToRun = std::move(I->second);211 AtExitRecords.erase(I);212 }213 }214 215 while (!AtExitsToRun.empty()) {216 AtExitsToRun.back().F(AtExitsToRun.back().Ctx);217 AtExitsToRun.pop_back();218 }219}220 221DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator(222 sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow,223 AddAbsoluteSymbolsFn AddAbsoluteSymbols)224 : Dylib(std::move(Dylib)), Allow(std::move(Allow)),225 AddAbsoluteSymbols(std::move(AddAbsoluteSymbols)),226 GlobalPrefix(GlobalPrefix) {}227 228Expected<std::unique_ptr<DynamicLibrarySearchGenerator>>229DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix,230 SymbolPredicate Allow,231 AddAbsoluteSymbolsFn AddAbsoluteSymbols) {232 std::string ErrMsg;233 auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg);234 if (!Lib.isValid())235 return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode());236 return std::make_unique<DynamicLibrarySearchGenerator>(237 std::move(Lib), GlobalPrefix, std::move(Allow),238 std::move(AddAbsoluteSymbols));239}240 241Error DynamicLibrarySearchGenerator::tryToGenerate(242 LookupState &LS, LookupKind K, JITDylib &JD,243 JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) {244 orc::SymbolMap NewSymbols;245 246 bool HasGlobalPrefix = (GlobalPrefix != '\0');247 248 for (auto &KV : Symbols) {249 auto &Name = KV.first;250 251 if ((*Name).empty())252 continue;253 254 if (Allow && !Allow(Name))255 continue;256 257 if (HasGlobalPrefix && (*Name).front() != GlobalPrefix)258 continue;259 260 std::string Tmp((*Name).data() + HasGlobalPrefix,261 (*Name).size() - HasGlobalPrefix);262 if (void *P = Dylib.getAddressOfSymbol(Tmp.c_str()))263 NewSymbols[Name] = {ExecutorAddr::fromPtr(P), JITSymbolFlags::Exported};264 }265 266 if (NewSymbols.empty())267 return Error::success();268 269 if (AddAbsoluteSymbols)270 return AddAbsoluteSymbols(JD, std::move(NewSymbols));271 return JD.define(absoluteSymbols(std::move(NewSymbols)));272}273 274StaticLibraryDefinitionGenerator::VisitMembersFunction275StaticLibraryDefinitionGenerator::loadAllObjectFileMembers(ObjectLayer &L,276 JITDylib &JD) {277 return [&](object::Archive &A, MemoryBufferRef Buf,278 size_t Index) -> Expected<bool> {279 switch (identify_magic(Buf.getBuffer())) {280 case file_magic::elf_relocatable:281 case file_magic::macho_object:282 case file_magic::coff_object:283 if (auto Err = L.add(JD, createMemberBuffer(A, Buf, Index)))284 return std::move(Err);285 // Since we've loaded it already, mark this as not loadable.286 return false;287 default:288 // Non-object-file members are not loadable.289 return false;290 }291 };292}293 294Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>295StaticLibraryDefinitionGenerator::Load(296 ObjectLayer &L, const char *FileName, VisitMembersFunction VisitMembers,297 GetObjectFileInterface GetObjFileInterface) {298 299 const auto &TT = L.getExecutionSession().getTargetTriple();300 auto Linkable = loadLinkableFile(FileName, TT, LoadArchives::Required);301 if (!Linkable)302 return Linkable.takeError();303 304 return Create(L, std::move(Linkable->first), std::move(VisitMembers),305 std::move(GetObjFileInterface));306}307 308Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>309StaticLibraryDefinitionGenerator::Create(310 ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,311 std::unique_ptr<object::Archive> Archive, VisitMembersFunction VisitMembers,312 GetObjectFileInterface GetObjFileInterface) {313 314 DenseSet<uint64_t> Excluded;315 316 if (VisitMembers) {317 size_t Index = 0;318 Error Err = Error::success();319 for (auto Child : Archive->children(Err)) {320 if (auto ChildBuf = Child.getMemoryBufferRef()) {321 if (auto Loadable = VisitMembers(*Archive, *ChildBuf, Index++)) {322 if (!*Loadable)323 Excluded.insert(Child.getDataOffset());324 } else325 return Loadable.takeError();326 } else {327 // We silently allow non-object archive members. This matches the328 // behavior of ld.329 consumeError(ChildBuf.takeError());330 }331 }332 if (Err)333 return std::move(Err);334 }335 336 DenseMap<SymbolStringPtr, size_t> SymbolToMemberIndexMap;337 {338 DenseMap<uint64_t, size_t> OffsetToIndex;339 size_t Index = 0;340 Error Err = Error::success();341 for (auto &Child : Archive->children(Err)) {342 // For all members not excluded above, add them to the OffsetToIndex map.343 if (!Excluded.count(Child.getDataOffset()))344 OffsetToIndex[Child.getDataOffset()] = Index;345 ++Index;346 }347 if (Err)348 return Err;349 350 auto &ES = L.getExecutionSession();351 for (auto &Sym : Archive->symbols()) {352 auto Member = Sym.getMember();353 if (!Member)354 return Member.takeError();355 auto EntryItr = OffsetToIndex.find(Member->getDataOffset());356 357 // Missing entry means this member should be ignored.358 if (EntryItr == OffsetToIndex.end())359 continue;360 361 SymbolToMemberIndexMap[ES.intern(Sym.getName())] = EntryItr->second;362 }363 }364 365 return std::unique_ptr<StaticLibraryDefinitionGenerator>(366 new StaticLibraryDefinitionGenerator(367 L, std::move(ArchiveBuffer), std::move(Archive),368 std::move(GetObjFileInterface), std::move(SymbolToMemberIndexMap)));369}370 371Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>372StaticLibraryDefinitionGenerator::Create(373 ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,374 VisitMembersFunction VisitMembers,375 GetObjectFileInterface GetObjFileInterface) {376 377 auto B = object::createBinary(ArchiveBuffer->getMemBufferRef());378 if (!B)379 return B.takeError();380 381 // If this is a regular archive then create an instance from it.382 if (isa<object::Archive>(*B))383 return Create(L, std::move(ArchiveBuffer),384 std::unique_ptr<object::Archive>(385 static_cast<object::Archive *>(B->release())),386 std::move(VisitMembers), std::move(GetObjFileInterface));387 388 // If this is a universal binary then search for a slice matching the given389 // Triple.390 if (auto *UB = dyn_cast<object::MachOUniversalBinary>(B->get())) {391 392 const auto &TT = L.getExecutionSession().getTargetTriple();393 394 auto SliceRange = getMachOSliceRangeForTriple(*UB, TT);395 if (!SliceRange)396 return SliceRange.takeError();397 398 MemoryBufferRef SliceRef(399 StringRef(ArchiveBuffer->getBufferStart() + SliceRange->first,400 SliceRange->second),401 ArchiveBuffer->getBufferIdentifier());402 403 auto Archive = object::Archive::create(SliceRef);404 if (!Archive)405 return Archive.takeError();406 407 return Create(L, std::move(ArchiveBuffer), std::move(*Archive),408 std::move(VisitMembers), std::move(GetObjFileInterface));409 }410 411 return make_error<StringError>(Twine("Unrecognized file type for ") +412 ArchiveBuffer->getBufferIdentifier(),413 inconvertibleErrorCode());414}415 416Error StaticLibraryDefinitionGenerator::tryToGenerate(417 LookupState &LS, LookupKind K, JITDylib &JD,418 JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) {419 // Don't materialize symbols from static archives unless this is a static420 // lookup.421 if (K != LookupKind::Static)422 return Error::success();423 424 // Bail out early if we've already freed the archive.425 if (!Archive)426 return Error::success();427 428 DenseMap<size_t, MemoryBufferRef> ToLoad;429 430 for (const auto &[Name, _] : Symbols) {431 // Check whehter the archive contains this symbol.432 auto It = SymbolToMemberIndexMap.find(Name);433 if (It == SymbolToMemberIndexMap.end())434 continue;435 size_t Index = It->second;436 437 // If we're already loading the member containing this symbol then we're438 // done.439 if (ToLoad.count(Index))440 continue;441 442 auto Member = Archive->findSym(*Name);443 if (!Member)444 return Member.takeError();445 if (!*Member) // Skip "none" children.446 continue;447 448 auto MemberBuf = (*Member)->getMemoryBufferRef();449 if (!MemberBuf)450 return MemberBuf.takeError();451 452 ToLoad[Index] = *MemberBuf;453 }454 455 // Remove symbols to be loaded.456 {457 // FIXME: Enable DenseMap removal using NonOwningSymbolStringPtr?458 std::vector<SymbolStringPtr> ToRemove;459 for (auto &[Name, Index] : SymbolToMemberIndexMap)460 if (ToLoad.count(Index))461 ToRemove.push_back(Name);462 for (auto &Name : ToRemove)463 SymbolToMemberIndexMap.erase(Name);464 }465 466 // Add loaded files to JITDylib.467 for (auto &[Index, Buf] : ToLoad) {468 auto MemberBuf = createMemberBuffer(*Archive, Buf, Index);469 470 auto Interface = GetObjFileInterface(L.getExecutionSession(),471 MemberBuf->getMemBufferRef());472 if (!Interface)473 return Interface.takeError();474 475 if (auto Err = L.add(JD, std::move(MemberBuf), std::move(*Interface)))476 return Err;477 }478 479 return Error::success();480}481 482std::unique_ptr<MemoryBuffer>483StaticLibraryDefinitionGenerator::createMemberBuffer(object::Archive &A,484 MemoryBufferRef BufRef,485 size_t Index) {486 return MemoryBuffer::getMemBuffer(BufRef.getBuffer(),487 (A.getFileName() + "[" + Twine(Index) +488 "](" + BufRef.getBufferIdentifier() + ")")489 .str(),490 false);491}492 493StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator(494 ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer,495 std::unique_ptr<object::Archive> Archive,496 GetObjectFileInterface GetObjFileInterface,497 DenseMap<SymbolStringPtr, size_t> SymbolToMemberIndexMap)498 : L(L), GetObjFileInterface(std::move(GetObjFileInterface)),499 ArchiveBuffer(std::move(ArchiveBuffer)), Archive(std::move(Archive)),500 SymbolToMemberIndexMap(std::move(SymbolToMemberIndexMap)) {501 if (!this->GetObjFileInterface)502 this->GetObjFileInterface = getObjectFileInterface;503}504 505std::unique_ptr<DLLImportDefinitionGenerator>506DLLImportDefinitionGenerator::Create(ExecutionSession &ES,507 ObjectLinkingLayer &L) {508 return std::unique_ptr<DLLImportDefinitionGenerator>(509 new DLLImportDefinitionGenerator(ES, L));510}511 512Error DLLImportDefinitionGenerator::tryToGenerate(513 LookupState &LS, LookupKind K, JITDylib &JD,514 JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) {515 JITDylibSearchOrder LinkOrder;516 JD.withLinkOrderDo([&](const JITDylibSearchOrder &LO) {517 LinkOrder.reserve(LO.size());518 for (auto &KV : LO) {519 if (KV.first == &JD)520 continue;521 LinkOrder.push_back(KV);522 }523 });524 525 // FIXME: if regular symbol name start with __imp_ we have to issue lookup of526 // both __imp_ and stripped name and use the lookup information to resolve the527 // real symbol name.528 SymbolLookupSet LookupSet;529 DenseMap<StringRef, SymbolLookupFlags> ToLookUpSymbols;530 for (auto &KV : Symbols) {531 StringRef Deinterned = *KV.first;532 if (Deinterned.starts_with(getImpPrefix()))533 Deinterned = Deinterned.drop_front(StringRef(getImpPrefix()).size());534 // Don't degrade the required state535 auto [It, Inserted] = ToLookUpSymbols.try_emplace(Deinterned);536 if (Inserted || It->second != SymbolLookupFlags::RequiredSymbol)537 It->second = KV.second;538 }539 540 for (auto &KV : ToLookUpSymbols)541 LookupSet.add(ES.intern(KV.first), KV.second);542 543 auto Resolved = ES.lookup(LinkOrder, LookupSet, LookupKind::Static,544 SymbolState::Resolved);545 if (!Resolved)546 return Resolved.takeError();547 548 auto G = createStubsGraph(*Resolved);549 if (!G)550 return G.takeError();551 return L.add(JD, std::move(*G));552}553 554Expected<std::unique_ptr<jitlink::LinkGraph>>555DLLImportDefinitionGenerator::createStubsGraph(const SymbolMap &Resolved) {556 auto G = std::make_unique<jitlink::LinkGraph>(557 "<DLLIMPORT_STUBS>", ES.getSymbolStringPool(), ES.getTargetTriple(),558 SubtargetFeatures(), jitlink::getGenericEdgeKindName);559 jitlink::Section &Sec =560 G->createSection(getSectionName(), MemProt::Read | MemProt::Exec);561 562 for (auto &KV : Resolved) {563 jitlink::Symbol &Target = G->addAbsoluteSymbol(564 *KV.first, KV.second.getAddress(), G->getPointerSize(),565 jitlink::Linkage::Strong, jitlink::Scope::Local, false);566 567 // Create __imp_ symbol568 jitlink::Symbol &Ptr =569 jitlink::x86_64::createAnonymousPointer(*G, Sec, &Target);570 Ptr.setName(G->intern((Twine(getImpPrefix()) + *KV.first).str()));571 Ptr.setLinkage(jitlink::Linkage::Strong);572 Ptr.setScope(jitlink::Scope::Default);573 574 // Create PLT stub575 // FIXME: check PLT stub of data symbol is not accessed576 jitlink::Block &StubBlock =577 jitlink::x86_64::createPointerJumpStubBlock(*G, Sec, Ptr);578 G->addDefinedSymbol(StubBlock, 0, *KV.first, StubBlock.getSize(),579 jitlink::Linkage::Strong, jitlink::Scope::Default, true,580 false);581 }582 583 return std::move(G);584}585 586} // End namespace orc.587} // End namespace llvm.588