497 lines · cpp
1//===--- JITLinkMemoryManager.cpp - JITLinkMemoryManager implementation ---===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h"10#include "llvm/ExecutionEngine/JITLink/JITLink.h"11#include "llvm/Support/FormatVariadic.h"12#include "llvm/Support/Process.h"13 14#define DEBUG_TYPE "jitlink"15 16using namespace llvm;17 18namespace llvm {19namespace jitlink {20 21JITLinkMemoryManager::~JITLinkMemoryManager() = default;22JITLinkMemoryManager::InFlightAlloc::~InFlightAlloc() = default;23 24BasicLayout::BasicLayout(LinkGraph &G) : G(G) {25 26 for (auto &Sec : G.sections()) {27 // Skip empty sections, and sections with NoAlloc lifetime policies.28 if (Sec.blocks().empty() ||29 Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)30 continue;31 32 auto &Seg = Segments[{Sec.getMemProt(), Sec.getMemLifetime()}];33 for (auto *B : Sec.blocks())34 if (LLVM_LIKELY(!B->isZeroFill()))35 Seg.ContentBlocks.push_back(B);36 else37 Seg.ZeroFillBlocks.push_back(B);38 }39 40 // Build Segments map.41 auto CompareBlocks = [](const Block *LHS, const Block *RHS) {42 // Sort by section, address and size43 if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal())44 return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal();45 if (LHS->getAddress() != RHS->getAddress())46 return LHS->getAddress() < RHS->getAddress();47 return LHS->getSize() < RHS->getSize();48 };49 50 LLVM_DEBUG(dbgs() << "Generated BasicLayout for " << G.getName() << ":\n");51 for (auto &KV : Segments) {52 auto &Seg = KV.second;53 54 llvm::sort(Seg.ContentBlocks, CompareBlocks);55 llvm::sort(Seg.ZeroFillBlocks, CompareBlocks);56 57 for (auto *B : Seg.ContentBlocks) {58 Seg.ContentSize = alignToBlock(Seg.ContentSize, *B);59 Seg.ContentSize += B->getSize();60 Seg.Alignment = std::max(Seg.Alignment, Align(B->getAlignment()));61 }62 63 uint64_t SegEndOffset = Seg.ContentSize;64 for (auto *B : Seg.ZeroFillBlocks) {65 SegEndOffset = alignToBlock(SegEndOffset, *B);66 SegEndOffset += B->getSize();67 Seg.Alignment = std::max(Seg.Alignment, Align(B->getAlignment()));68 }69 Seg.ZeroFillSize = SegEndOffset - Seg.ContentSize;70 71 LLVM_DEBUG({72 dbgs() << " Seg " << KV.first73 << ": content-size=" << formatv("{0:x}", Seg.ContentSize)74 << ", zero-fill-size=" << formatv("{0:x}", Seg.ZeroFillSize)75 << ", align=" << formatv("{0:x}", Seg.Alignment.value()) << "\n";76 });77 }78}79 80Expected<BasicLayout::ContiguousPageBasedLayoutSizes>81BasicLayout::getContiguousPageBasedLayoutSizes(uint64_t PageSize) {82 ContiguousPageBasedLayoutSizes SegsSizes;83 84 for (auto &KV : segments()) {85 auto &AG = KV.first;86 auto &Seg = KV.second;87 88 if (Seg.Alignment > PageSize)89 return make_error<StringError>("Segment alignment greater than page size",90 inconvertibleErrorCode());91 92 uint64_t SegSize = alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize);93 if (AG.getMemLifetime() == orc::MemLifetime::Standard)94 SegsSizes.StandardSegs += SegSize;95 else96 SegsSizes.FinalizeSegs += SegSize;97 }98 99 return SegsSizes;100}101 102Error BasicLayout::apply() {103 for (auto &KV : Segments) {104 auto &Seg = KV.second;105 106 assert(!(Seg.ContentBlocks.empty() && Seg.ZeroFillBlocks.empty()) &&107 "Empty section recorded?");108 109 for (auto *B : Seg.ContentBlocks) {110 // Align addr and working-mem-offset.111 Seg.Addr = alignToBlock(Seg.Addr, *B);112 Seg.NextWorkingMemOffset = alignToBlock(Seg.NextWorkingMemOffset, *B);113 114 // Update block addr.115 B->setAddress(Seg.Addr);116 Seg.Addr += B->getSize();117 118 // Copy content to working memory, then update content to point at working119 // memory.120 memcpy(Seg.WorkingMem + Seg.NextWorkingMemOffset, B->getContent().data(),121 B->getSize());122 B->setMutableContent(123 {Seg.WorkingMem + Seg.NextWorkingMemOffset, B->getSize()});124 Seg.NextWorkingMemOffset += B->getSize();125 }126 127 for (auto *B : Seg.ZeroFillBlocks) {128 // Align addr.129 Seg.Addr = alignToBlock(Seg.Addr, *B);130 // Update block addr.131 B->setAddress(Seg.Addr);132 Seg.Addr += B->getSize();133 }134 135 Seg.ContentBlocks.clear();136 Seg.ZeroFillBlocks.clear();137 }138 139 return Error::success();140}141 142orc::shared::AllocActions &BasicLayout::graphAllocActions() {143 return G.allocActions();144}145 146void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr,147 std::shared_ptr<orc::SymbolStringPool> SSP,148 Triple TT, const JITLinkDylib *JD,149 SegmentMap Segments,150 OnCreatedFunction OnCreated) {151 152 static_assert(orc::AllocGroup::NumGroups == 32,153 "AllocGroup has changed. Section names below must be updated");154 StringRef AGSectionNames[] = {155 "__---.standard", "__R--.standard", "__-W-.standard", "__RW-.standard",156 "__--X.standard", "__R-X.standard", "__-WX.standard", "__RWX.standard",157 "__---.finalize", "__R--.finalize", "__-W-.finalize", "__RW-.finalize",158 "__--X.finalize", "__R-X.finalize", "__-WX.finalize", "__RWX.finalize"};159 160 auto G =161 std::make_unique<LinkGraph>("", std::move(SSP), std::move(TT),162 SubtargetFeatures(), getGenericEdgeKindName);163 orc::AllocGroupSmallMap<Block *> ContentBlocks;164 165 orc::ExecutorAddr NextAddr(0x100000);166 for (auto &KV : Segments) {167 auto &AG = KV.first;168 auto &Seg = KV.second;169 170 assert(AG.getMemLifetime() != orc::MemLifetime::NoAlloc &&171 "NoAlloc segments are not supported by SimpleSegmentAlloc");172 173 auto AGSectionName =174 AGSectionNames[static_cast<unsigned>(AG.getMemProt()) |175 static_cast<bool>(AG.getMemLifetime()) << 3];176 177 auto &Sec = G->createSection(AGSectionName, AG.getMemProt());178 Sec.setMemLifetime(AG.getMemLifetime());179 180 if (Seg.ContentSize != 0) {181 NextAddr =182 orc::ExecutorAddr(alignTo(NextAddr.getValue(), Seg.ContentAlign));183 auto &B =184 G->createMutableContentBlock(Sec, G->allocateBuffer(Seg.ContentSize),185 NextAddr, Seg.ContentAlign.value(), 0);186 ContentBlocks[AG] = &B;187 NextAddr += Seg.ContentSize;188 }189 }190 191 // GRef declared separately since order-of-argument-eval isn't specified.192 auto &GRef = *G;193 MemMgr.allocate(JD, GRef,194 [G = std::move(G), ContentBlocks = std::move(ContentBlocks),195 OnCreated = std::move(OnCreated)](196 JITLinkMemoryManager::AllocResult Alloc) mutable {197 if (!Alloc)198 OnCreated(Alloc.takeError());199 else200 OnCreated(SimpleSegmentAlloc(std::move(G),201 std::move(ContentBlocks),202 std::move(*Alloc)));203 });204}205 206Expected<SimpleSegmentAlloc> SimpleSegmentAlloc::Create(207 JITLinkMemoryManager &MemMgr, std::shared_ptr<orc::SymbolStringPool> SSP,208 Triple TT, const JITLinkDylib *JD, SegmentMap Segments) {209 std::promise<MSVCPExpected<SimpleSegmentAlloc>> AllocP;210 auto AllocF = AllocP.get_future();211 Create(MemMgr, std::move(SSP), std::move(TT), JD, std::move(Segments),212 [&](Expected<SimpleSegmentAlloc> Result) {213 AllocP.set_value(std::move(Result));214 });215 return AllocF.get();216}217 218SimpleSegmentAlloc::SimpleSegmentAlloc(SimpleSegmentAlloc &&) = default;219SimpleSegmentAlloc &220SimpleSegmentAlloc::operator=(SimpleSegmentAlloc &&) = default;221SimpleSegmentAlloc::~SimpleSegmentAlloc() = default;222 223SimpleSegmentAlloc::SegmentInfo224SimpleSegmentAlloc::getSegInfo(orc::AllocGroup AG) {225 auto I = ContentBlocks.find(AG);226 if (I != ContentBlocks.end()) {227 auto &B = *I->second;228 return {B.getAddress(), B.getAlreadyMutableContent()};229 }230 return {};231}232 233SimpleSegmentAlloc::SimpleSegmentAlloc(234 std::unique_ptr<LinkGraph> G,235 orc::AllocGroupSmallMap<Block *> ContentBlocks,236 std::unique_ptr<JITLinkMemoryManager::InFlightAlloc> Alloc)237 : G(std::move(G)), ContentBlocks(std::move(ContentBlocks)),238 Alloc(std::move(Alloc)) {}239 240class InProcessMemoryManager::IPInFlightAlloc241 : public JITLinkMemoryManager::InFlightAlloc {242public:243 IPInFlightAlloc(InProcessMemoryManager &MemMgr, LinkGraph &G, BasicLayout BL,244 sys::MemoryBlock StandardSegments,245 sys::MemoryBlock FinalizationSegments)246 : MemMgr(MemMgr), G(&G), BL(std::move(BL)),247 StandardSegments(std::move(StandardSegments)),248 FinalizationSegments(std::move(FinalizationSegments)) {}249 250 ~IPInFlightAlloc() override {251 assert(!G && "InFlight alloc neither abandoned nor finalized");252 }253 254 void finalize(OnFinalizedFunction OnFinalized) override {255 256 // Apply memory protections to all segments.257 if (auto Err = applyProtections()) {258 OnFinalized(std::move(Err));259 return;260 }261 262 // Run finalization actions.263 auto DeallocActions = runFinalizeActions(G->allocActions());264 if (!DeallocActions) {265 OnFinalized(DeallocActions.takeError());266 return;267 }268 269 // Release the finalize segments slab.270 if (auto EC = sys::Memory::releaseMappedMemory(FinalizationSegments)) {271 OnFinalized(errorCodeToError(EC));272 return;273 }274 275#ifndef NDEBUG276 // Set 'G' to null to flag that we've been successfully finalized.277 // This allows us to assert at destruction time that a call has been made278 // to either finalize or abandon.279 G = nullptr;280#endif281 282 // Continue with finalized allocation.283 OnFinalized(MemMgr.createFinalizedAlloc(std::move(StandardSegments),284 std::move(*DeallocActions)));285 }286 287 void abandon(OnAbandonedFunction OnAbandoned) override {288 Error Err = Error::success();289 if (auto EC = sys::Memory::releaseMappedMemory(FinalizationSegments))290 Err = joinErrors(std::move(Err), errorCodeToError(EC));291 if (auto EC = sys::Memory::releaseMappedMemory(StandardSegments))292 Err = joinErrors(std::move(Err), errorCodeToError(EC));293 294#ifndef NDEBUG295 // Set 'G' to null to flag that we've been successfully finalized.296 // This allows us to assert at destruction time that a call has been made297 // to either finalize or abandon.298 G = nullptr;299#endif300 301 OnAbandoned(std::move(Err));302 }303 304private:305 Error applyProtections() {306 for (auto &KV : BL.segments()) {307 const auto &AG = KV.first;308 auto &Seg = KV.second;309 310 auto Prot = toSysMemoryProtectionFlags(AG.getMemProt());311 312 uint64_t SegSize =313 alignTo(Seg.ContentSize + Seg.ZeroFillSize, MemMgr.PageSize);314 sys::MemoryBlock MB(Seg.WorkingMem, SegSize);315 if (auto EC = sys::Memory::protectMappedMemory(MB, Prot))316 return errorCodeToError(EC);317 if (Prot & sys::Memory::MF_EXEC)318 sys::Memory::InvalidateInstructionCache(MB.base(), MB.allocatedSize());319 }320 return Error::success();321 }322 323 InProcessMemoryManager &MemMgr;324 LinkGraph *G;325 BasicLayout BL;326 sys::MemoryBlock StandardSegments;327 sys::MemoryBlock FinalizationSegments;328};329 330Expected<std::unique_ptr<InProcessMemoryManager>>331InProcessMemoryManager::Create() {332 if (auto PageSize = sys::Process::getPageSize()) {333 // FIXME: Just check this once on startup.334 if (!isPowerOf2_64((uint64_t)*PageSize))335 return make_error<StringError>(336 "Could not create InProcessMemoryManager: Page size " +337 Twine(*PageSize) + " is not a power of 2",338 inconvertibleErrorCode());339 340 return std::make_unique<InProcessMemoryManager>(*PageSize);341 } else342 return PageSize.takeError();343}344 345void InProcessMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G,346 OnAllocatedFunction OnAllocated) {347 BasicLayout BL(G);348 349 /// Scan the request and calculate the group and total sizes.350 /// Check that segment size is no larger than a page.351 auto SegsSizes = BL.getContiguousPageBasedLayoutSizes(PageSize);352 if (!SegsSizes) {353 OnAllocated(SegsSizes.takeError());354 return;355 }356 357 /// Check that the total size requested (including zero fill) is not larger358 /// than a size_t.359 if (SegsSizes->total() > std::numeric_limits<size_t>::max()) {360 OnAllocated(make_error<JITLinkError>(361 "Total requested size " + formatv("{0:x}", SegsSizes->total()) +362 " for graph " + G.getName() + " exceeds address space"));363 return;364 }365 366 // Allocate one slab for the whole thing (to make sure everything is367 // in-range), then partition into standard and finalization blocks.368 //369 // FIXME: Make two separate allocations in the future to reduce370 // fragmentation: finalization segments will usually be a single page, and371 // standard segments are likely to be more than one page. Where multiple372 // allocations are in-flight at once (likely) the current approach will leave373 // a lot of single-page holes.374 sys::MemoryBlock Slab;375 sys::MemoryBlock StandardSegsMem;376 sys::MemoryBlock FinalizeSegsMem;377 {378 const sys::Memory::ProtectionFlags ReadWrite =379 static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |380 sys::Memory::MF_WRITE);381 382 std::error_code EC;383 Slab = sys::Memory::allocateMappedMemory(SegsSizes->total(), nullptr,384 ReadWrite, EC);385 386 if (EC) {387 OnAllocated(errorCodeToError(EC));388 return;389 }390 391 // Zero-fill the whole slab up-front.392 memset(Slab.base(), 0, Slab.allocatedSize());393 394 StandardSegsMem = {Slab.base(),395 static_cast<size_t>(SegsSizes->StandardSegs)};396 FinalizeSegsMem = {(void *)((char *)Slab.base() + SegsSizes->StandardSegs),397 static_cast<size_t>(SegsSizes->FinalizeSegs)};398 }399 400 auto NextStandardSegAddr = orc::ExecutorAddr::fromPtr(StandardSegsMem.base());401 auto NextFinalizeSegAddr = orc::ExecutorAddr::fromPtr(FinalizeSegsMem.base());402 403 LLVM_DEBUG({404 dbgs() << "InProcessMemoryManager allocated:\n";405 if (SegsSizes->StandardSegs)406 dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextStandardSegAddr,407 NextStandardSegAddr + StandardSegsMem.allocatedSize())408 << " to stardard segs\n";409 else410 dbgs() << " no standard segs\n";411 if (SegsSizes->FinalizeSegs)412 dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextFinalizeSegAddr,413 NextFinalizeSegAddr + FinalizeSegsMem.allocatedSize())414 << " to finalize segs\n";415 else416 dbgs() << " no finalize segs\n";417 });418 419 // Build ProtMap, assign addresses.420 for (auto &KV : BL.segments()) {421 auto &AG = KV.first;422 auto &Seg = KV.second;423 424 auto &SegAddr = (AG.getMemLifetime() == orc::MemLifetime::Standard)425 ? NextStandardSegAddr426 : NextFinalizeSegAddr;427 428 Seg.WorkingMem = SegAddr.toPtr<char *>();429 Seg.Addr = SegAddr;430 431 SegAddr += alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize);432 }433 434 if (auto Err = BL.apply()) {435 OnAllocated(std::move(Err));436 return;437 }438 439 OnAllocated(std::make_unique<IPInFlightAlloc>(*this, G, std::move(BL),440 std::move(StandardSegsMem),441 std::move(FinalizeSegsMem)));442}443 444void InProcessMemoryManager::deallocate(std::vector<FinalizedAlloc> Allocs,445 OnDeallocatedFunction OnDeallocated) {446 std::vector<sys::MemoryBlock> StandardSegmentsList;447 std::vector<std::vector<orc::shared::WrapperFunctionCall>> DeallocActionsList;448 449 {450 std::lock_guard<std::mutex> Lock(FinalizedAllocsMutex);451 for (auto &Alloc : Allocs) {452 auto *FA = Alloc.release().toPtr<FinalizedAllocInfo *>();453 StandardSegmentsList.push_back(std::move(FA->StandardSegments));454 DeallocActionsList.push_back(std::move(FA->DeallocActions));455 FA->~FinalizedAllocInfo();456 FinalizedAllocInfos.Deallocate(FA);457 }458 }459 460 Error DeallocErr = Error::success();461 462 while (!DeallocActionsList.empty()) {463 auto &DeallocActions = DeallocActionsList.back();464 auto &StandardSegments = StandardSegmentsList.back();465 466 /// Run any deallocate calls.467 while (!DeallocActions.empty()) {468 if (auto Err = DeallocActions.back().runWithSPSRetErrorMerged())469 DeallocErr = joinErrors(std::move(DeallocErr), std::move(Err));470 DeallocActions.pop_back();471 }472 473 /// Release the standard segments slab.474 if (auto EC = sys::Memory::releaseMappedMemory(StandardSegments))475 DeallocErr = joinErrors(std::move(DeallocErr), errorCodeToError(EC));476 477 DeallocActionsList.pop_back();478 StandardSegmentsList.pop_back();479 }480 481 OnDeallocated(std::move(DeallocErr));482}483 484JITLinkMemoryManager::FinalizedAlloc485InProcessMemoryManager::createFinalizedAlloc(486 sys::MemoryBlock StandardSegments,487 std::vector<orc::shared::WrapperFunctionCall> DeallocActions) {488 std::lock_guard<std::mutex> Lock(FinalizedAllocsMutex);489 auto *FA = FinalizedAllocInfos.Allocate<FinalizedAllocInfo>();490 new (FA) FinalizedAllocInfo(491 {std::move(StandardSegments), std::move(DeallocActions)});492 return FinalizedAlloc(orc::ExecutorAddr::fromPtr(FA));493}494 495} // end namespace jitlink496} // end namespace llvm497