430 lines · cpp
1//===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h"10 11#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h"12#include "llvm/Support/MathExtras.h"13 14#include <future>15 16using namespace llvm;17using namespace llvm::orc;18 19namespace llvm {20namespace orc {21 22class EPCIndirectionUtilsAccess {23public:24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;26 27 static Expected<IndirectStubInfoVector>28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {29 return EPCIU.getIndirectStubs(NumStubs);30 };31};32 33} // end namespace orc34} // end namespace llvm35 36namespace {37 38class EPCTrampolinePool : public TrampolinePool {39public:40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU);41 Error deallocatePool();42 43protected:44 Error grow() override;45 46 using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc;47 48 EPCIndirectionUtils &EPCIU;49 unsigned TrampolineSize = 0;50 unsigned TrampolinesPerPage = 0;51 std::vector<FinalizedAlloc> TrampolineBlocks;52};53 54class EPCIndirectStubsManager : public IndirectStubsManager,55 private EPCIndirectionUtilsAccess {56public:57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}58 59 Error deallocateStubs();60 61 Error createStub(StringRef StubName, ExecutorAddr StubAddr,62 JITSymbolFlags StubFlags) override;63 64 Error createStubs(const StubInitsMap &StubInits) override;65 66 ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override;67 68 ExecutorSymbolDef findPointer(StringRef Name) override;69 70 Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override;71 72private:73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;74 75 std::mutex ISMMutex;76 EPCIndirectionUtils &EPCIU;77 StringMap<StubInfo> StubInfos;78};79 80EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)81 : EPCIU(EPCIU) {82 auto &EPC = EPCIU.getExecutorProcessControl();83 auto &ABI = EPCIU.getABISupport();84 85 TrampolineSize = ABI.getTrampolineSize();86 TrampolinesPerPage =87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;88}89 90Error EPCTrampolinePool::deallocatePool() {91 std::promise<MSVCPError> DeallocResultP;92 auto DeallocResultF = DeallocResultP.get_future();93 94 EPCIU.getExecutorProcessControl().getMemMgr().deallocate(95 std::move(TrampolineBlocks),96 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });97 98 return DeallocResultF.get();99}100 101Error EPCTrampolinePool::grow() {102 using namespace jitlink;103 104 assert(AvailableTrampolines.empty() &&105 "Grow called with trampolines still available");106 107 auto ResolverAddress = EPCIU.getResolverBlockAddress();108 assert(ResolverAddress && "Resolver address can not be null");109 110 auto &EPC = EPCIU.getExecutorProcessControl();111 auto PageSize = EPC.getPageSize();112 auto Alloc = SimpleSegmentAlloc::Create(113 EPC.getMemMgr(), EPC.getSymbolStringPool(), EPC.getTargetTriple(),114 nullptr, {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});115 if (!Alloc)116 return Alloc.takeError();117 118 unsigned NumTrampolines = TrampolinesPerPage;119 120 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);121 EPCIU.getABISupport().writeTrampolines(122 SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines);123 for (unsigned I = 0; I < NumTrampolines; ++I)124 AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize));125 126 auto FA = Alloc->finalize();127 if (!FA)128 return FA.takeError();129 130 TrampolineBlocks.push_back(std::move(*FA));131 132 return Error::success();133}134 135Error EPCIndirectStubsManager::createStub(StringRef StubName,136 ExecutorAddr StubAddr,137 JITSymbolFlags StubFlags) {138 StubInitsMap SIM;139 SIM[StubName] = std::make_pair(StubAddr, StubFlags);140 return createStubs(SIM);141}142 143Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {144 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());145 if (!AvailableStubInfos)146 return AvailableStubInfos.takeError();147 148 {149 std::lock_guard<std::mutex> Lock(ISMMutex);150 unsigned ASIdx = 0;151 for (auto &SI : StubInits) {152 auto &A = (*AvailableStubInfos)[ASIdx++];153 StubInfos[SI.first()] = std::make_pair(A, SI.second.second);154 }155 }156 157 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();158 switch (EPCIU.getABISupport().getPointerSize()) {159 case 4: {160 unsigned ASIdx = 0;161 std::vector<tpctypes::UInt32Write> PtrUpdates;162 for (auto &SI : StubInits)163 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,164 static_cast<uint32_t>(SI.second.first.getValue())});165 return MemAccess.writeUInt32s(PtrUpdates);166 }167 case 8: {168 unsigned ASIdx = 0;169 std::vector<tpctypes::UInt64Write> PtrUpdates;170 for (auto &SI : StubInits)171 PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,172 SI.second.first.getValue()});173 return MemAccess.writeUInt64s(PtrUpdates);174 }175 default:176 return make_error<StringError>("Unsupported pointer size",177 inconvertibleErrorCode());178 }179}180 181ExecutorSymbolDef EPCIndirectStubsManager::findStub(StringRef Name,182 bool ExportedStubsOnly) {183 std::lock_guard<std::mutex> Lock(ISMMutex);184 auto I = StubInfos.find(Name);185 if (I == StubInfos.end())186 return ExecutorSymbolDef();187 return {I->second.first.StubAddress, I->second.second};188}189 190ExecutorSymbolDef EPCIndirectStubsManager::findPointer(StringRef Name) {191 std::lock_guard<std::mutex> Lock(ISMMutex);192 auto I = StubInfos.find(Name);193 if (I == StubInfos.end())194 return ExecutorSymbolDef();195 return {I->second.first.PointerAddress, I->second.second};196}197 198Error EPCIndirectStubsManager::updatePointer(StringRef Name,199 ExecutorAddr NewAddr) {200 201 ExecutorAddr PtrAddr;202 {203 std::lock_guard<std::mutex> Lock(ISMMutex);204 auto I = StubInfos.find(Name);205 if (I == StubInfos.end())206 return make_error<StringError>("Unknown stub name",207 inconvertibleErrorCode());208 PtrAddr = I->second.first.PointerAddress;209 }210 211 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();212 switch (EPCIU.getABISupport().getPointerSize()) {213 case 4: {214 tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr.getValue());215 return MemAccess.writeUInt32s(PUpdate);216 }217 case 8: {218 tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr.getValue());219 return MemAccess.writeUInt64s(PUpdate);220 }221 default:222 return make_error<StringError>("Unsupported pointer size",223 inconvertibleErrorCode());224 }225}226 227} // end anonymous namespace.228 229namespace llvm {230namespace orc {231 232EPCIndirectionUtils::ABISupport::~ABISupport() = default;233 234Expected<std::unique_ptr<EPCIndirectionUtils>>235EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) {236 const auto &TT = EPC.getTargetTriple();237 switch (TT.getArch()) {238 default:239 return make_error<StringError>(240 std::string("No EPCIndirectionUtils available for ") + TT.str(),241 inconvertibleErrorCode());242 case Triple::aarch64:243 case Triple::aarch64_32:244 return CreateWithABI<OrcAArch64>(EPC);245 246 case Triple::x86:247 return CreateWithABI<OrcI386>(EPC);248 249 case Triple::loongarch64:250 return CreateWithABI<OrcLoongArch64>(EPC);251 252 case Triple::mips:253 return CreateWithABI<OrcMips32Be>(EPC);254 255 case Triple::mipsel:256 return CreateWithABI<OrcMips32Le>(EPC);257 258 case Triple::mips64:259 case Triple::mips64el:260 return CreateWithABI<OrcMips64>(EPC);261 262 case Triple::riscv64:263 return CreateWithABI<OrcRiscv64>(EPC);264 265 case Triple::x86_64:266 if (TT.getOS() == Triple::OSType::Win32)267 return CreateWithABI<OrcX86_64_Win32>(EPC);268 else269 return CreateWithABI<OrcX86_64_SysV>(EPC);270 }271}272 273Error EPCIndirectionUtils::cleanup() {274 275 auto &MemMgr = EPC.getMemMgr();276 auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));277 278 if (TP)279 Err = joinErrors(std::move(Err),280 static_cast<EPCTrampolinePool &>(*TP).deallocatePool());281 282 if (ResolverBlock)283 Err =284 joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));285 286 return Err;287}288 289Expected<ExecutorAddr>290EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr,291 ExecutorAddr ReentryCtxAddr) {292 using namespace jitlink;293 294 assert(ABI && "ABI can not be null");295 auto ResolverSize = ABI->getResolverCodeSize();296 297 auto Alloc =298 SimpleSegmentAlloc::Create(EPC.getMemMgr(), EPC.getSymbolStringPool(),299 EPC.getTargetTriple(), nullptr,300 {{MemProt::Read | MemProt::Exec,301 {ResolverSize, Align(EPC.getPageSize())}}});302 303 if (!Alloc)304 return Alloc.takeError();305 306 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);307 ResolverBlockAddr = SegInfo.Addr;308 ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr,309 ReentryFnAddr, ReentryCtxAddr);310 311 auto FA = Alloc->finalize();312 if (!FA)313 return FA.takeError();314 315 ResolverBlock = std::move(*FA);316 return ResolverBlockAddr;317}318 319std::unique_ptr<IndirectStubsManager>320EPCIndirectionUtils::createIndirectStubsManager() {321 return std::make_unique<EPCIndirectStubsManager>(*this);322}323 324TrampolinePool &EPCIndirectionUtils::getTrampolinePool() {325 if (!TP)326 TP = std::make_unique<EPCTrampolinePool>(*this);327 return *TP;328}329 330LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager(331 ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr) {332 assert(!LCTM &&333 "createLazyCallThroughManager can not have been called before");334 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,335 &getTrampolinePool());336 return *LCTM;337}338 339EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,340 std::unique_ptr<ABISupport> ABI)341 : EPC(EPC), ABI(std::move(ABI)) {342 assert(this->ABI && "ABI can not be null");343 344 assert(EPC.getPageSize() > getABISupport().getStubSize() &&345 "Stubs larger than one page are not supported");346}347 348Expected<EPCIndirectionUtils::IndirectStubInfoVector>349EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {350 using namespace jitlink;351 352 std::lock_guard<std::mutex> Lock(EPCUIMutex);353 354 // If there aren't enough stubs available then allocate some more.355 if (NumStubs > AvailableIndirectStubs.size()) {356 auto NumStubsToAllocate = NumStubs;357 auto PageSize = EPC.getPageSize();358 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);359 NumStubsToAllocate = StubBytes / ABI->getStubSize();360 auto PtrBytes =361 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);362 363 auto StubProt = MemProt::Read | MemProt::Exec;364 auto PtrProt = MemProt::Read | MemProt::Write;365 366 auto Alloc = SimpleSegmentAlloc::Create(367 EPC.getMemMgr(), EPC.getSymbolStringPool(), EPC.getTargetTriple(),368 nullptr,369 {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},370 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});371 372 if (!Alloc)373 return Alloc.takeError();374 375 auto StubSeg = Alloc->getSegInfo(StubProt);376 auto PtrSeg = Alloc->getSegInfo(PtrProt);377 378 ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr,379 PtrSeg.Addr, NumStubsToAllocate);380 381 auto FA = Alloc->finalize();382 if (!FA)383 return FA.takeError();384 385 IndirectStubAllocs.push_back(std::move(*FA));386 387 auto StubExecutorAddr = StubSeg.Addr;388 auto PtrExecutorAddr = PtrSeg.Addr;389 for (unsigned I = 0; I != NumStubsToAllocate; ++I) {390 AvailableIndirectStubs.push_back(391 IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr));392 StubExecutorAddr += ABI->getStubSize();393 PtrExecutorAddr += ABI->getPointerSize();394 }395 }396 397 assert(NumStubs <= AvailableIndirectStubs.size() &&398 "Sufficient stubs should have been allocated above");399 400 IndirectStubInfoVector Result;401 while (NumStubs--) {402 Result.push_back(AvailableIndirectStubs.back());403 AvailableIndirectStubs.pop_back();404 }405 406 return std::move(Result);407}408 409static JITTargetAddress reentry(JITTargetAddress LCTMAddr,410 JITTargetAddress TrampolineAddr) {411 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);412 std::promise<ExecutorAddr> LandingAddrP;413 auto LandingAddrF = LandingAddrP.get_future();414 LCTM.resolveTrampolineLandingAddress(415 ExecutorAddr(TrampolineAddr),416 [&](ExecutorAddr Addr) { LandingAddrP.set_value(Addr); });417 return LandingAddrF.get().getValue();418}419 420Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) {421 auto &LCTM = EPCIU.getLazyCallThroughManager();422 return EPCIU423 .writeResolverBlock(ExecutorAddr::fromPtr(&reentry),424 ExecutorAddr::fromPtr(&LCTM))425 .takeError();426}427 428} // end namespace orc429} // end namespace llvm430