276 lines · cpp
1//===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the section-based memory manager used by the MCJIT10// execution engine and RuntimeDyld11//12//===----------------------------------------------------------------------===//13 14#include "llvm/ExecutionEngine/SectionMemoryManager.h"15#include "llvm/Config/config.h"16#include "llvm/Support/Process.h"17 18namespace llvm {19 20uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size,21 unsigned Alignment,22 unsigned SectionID,23 StringRef SectionName,24 bool IsReadOnly) {25 if (IsReadOnly)26 return allocateSection(SectionMemoryManager::AllocationPurpose::ROData,27 Size, Alignment);28 return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size,29 Alignment);30}31 32uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size,33 unsigned Alignment,34 unsigned SectionID,35 StringRef SectionName) {36 return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size,37 Alignment);38}39 40uint8_t *SectionMemoryManager::allocateSection(41 SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size,42 unsigned Alignment) {43 if (!Alignment)44 Alignment = 16;45 46 assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two.");47 48 uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1);49 uintptr_t Addr = 0;50 51 MemoryGroup &MemGroup = [&]() -> MemoryGroup & {52 switch (Purpose) {53 case AllocationPurpose::Code:54 return CodeMem;55 case AllocationPurpose::ROData:56 return RODataMem;57 case AllocationPurpose::RWData:58 return RWDataMem;59 }60 llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose");61 }();62 63 // Look in the list of free memory regions and use a block there if one64 // is available.65 for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {66 if (FreeMB.Free.allocatedSize() >= RequiredSize) {67 Addr = (uintptr_t)FreeMB.Free.base();68 uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize();69 // Align the address.70 Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);71 72 if (FreeMB.PendingPrefixIndex == (unsigned)-1) {73 // The part of the block we're giving out to the user is now pending74 MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));75 76 // Remember this pending block, such that future allocations can just77 // modify it rather than creating a new one78 FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;79 } else {80 sys::MemoryBlock &PendingMB =81 MemGroup.PendingMem[FreeMB.PendingPrefixIndex];82 PendingMB = sys::MemoryBlock(PendingMB.base(),83 Addr + Size - (uintptr_t)PendingMB.base());84 }85 86 // Remember how much free space is now left in this block87 FreeMB.Free =88 sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);89 return (uint8_t *)Addr;90 }91 }92 93 // No pre-allocated free block was large enough. Allocate a new memory region.94 // Note that all sections get allocated as read-write. The permissions will95 // be updated later based on memory group.96 //97 // FIXME: It would be useful to define a default allocation size (or add98 // it as a constructor parameter) to minimize the number of allocations.99 //100 // FIXME: Initialize the Near member for each memory group to avoid101 // interleaving.102 std::error_code ec;103 sys::MemoryBlock MB = MMapper->allocateMappedMemory(104 Purpose, RequiredSize, &MemGroup.Near,105 sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec);106 if (ec) {107 // FIXME: Add error propagation to the interface.108 return nullptr;109 }110 111 // Save this address as the basis for our next request112 MemGroup.Near = MB;113 114 // Copy the address to all the other groups, if they have not115 // been initialized.116 if (CodeMem.Near.base() == nullptr)117 CodeMem.Near = MB;118 if (RODataMem.Near.base() == nullptr)119 RODataMem.Near = MB;120 if (RWDataMem.Near.base() == nullptr)121 RWDataMem.Near = MB;122 123 // Remember that we allocated this memory124 MemGroup.AllocatedMem.push_back(MB);125 Addr = (uintptr_t)MB.base();126 uintptr_t EndOfBlock = Addr + MB.allocatedSize();127 128 // Align the address.129 Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);130 131 // The part of the block we're giving out to the user is now pending132 MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));133 134 // The allocateMappedMemory may allocate much more memory than we need. In135 // this case, we store the unused memory as a free memory block.136 unsigned FreeSize = EndOfBlock - Addr - Size;137 if (FreeSize > 16) {138 FreeMemBlock FreeMB;139 FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize);140 FreeMB.PendingPrefixIndex = (unsigned)-1;141 MemGroup.FreeMem.push_back(FreeMB);142 }143 144 // Return aligned address145 return (uint8_t *)Addr;146}147 148bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) {149 // FIXME: Should in-progress permissions be reverted if an error occurs?150 std::error_code ec;151 152 // Make code memory executable.153 ec = applyMemoryGroupPermissions(CodeMem,154 sys::Memory::MF_READ | sys::Memory::MF_EXEC);155 if (ec) {156 if (ErrMsg) {157 *ErrMsg = ec.message();158 }159 return true;160 }161 162 // Make read-only data memory read-only.163 ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ);164 if (ec) {165 if (ErrMsg) {166 *ErrMsg = ec.message();167 }168 return true;169 }170 171 // Read-write data memory already has the correct permissions172 173 // Some platforms with separate data cache and instruction cache require174 // explicit cache flush, otherwise JIT code manipulations (like resolved175 // relocations) will get to the data cache but not to the instruction cache.176 invalidateInstructionCache();177 178 return false;179}180 181static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {182 static const size_t PageSize = sys::Process::getPageSizeEstimate();183 184 size_t StartOverlap =185 (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;186 187 size_t TrimmedSize = M.allocatedSize();188 TrimmedSize -= StartOverlap;189 TrimmedSize -= TrimmedSize % PageSize;190 191 sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap),192 TrimmedSize);193 194 assert(((uintptr_t)Trimmed.base() % PageSize) == 0);195 assert((Trimmed.allocatedSize() % PageSize) == 0);196 assert(M.base() <= Trimmed.base() &&197 Trimmed.allocatedSize() <= M.allocatedSize());198 199 return Trimmed;200}201 202std::error_code203SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,204 unsigned Permissions) {205 for (sys::MemoryBlock &MB : MemGroup.PendingMem)206 if (std::error_code EC = MMapper->protectMappedMemory(MB, Permissions))207 return EC;208 209 MemGroup.PendingMem.clear();210 211 // Now go through free blocks and trim any of them that don't span the entire212 // page because one of the pending blocks may have overlapped it.213 for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {214 FreeMB.Free = trimBlockToPageSize(FreeMB.Free);215 // We cleared the PendingMem list, so all these pointers are now invalid216 FreeMB.PendingPrefixIndex = (unsigned)-1;217 }218 219 // Remove all blocks which are now empty220 erase_if(MemGroup.FreeMem, [](FreeMemBlock &FreeMB) {221 return FreeMB.Free.allocatedSize() == 0;222 });223 224 return std::error_code();225}226 227void SectionMemoryManager::invalidateInstructionCache() {228 for (sys::MemoryBlock &Block : CodeMem.PendingMem)229 sys::Memory::InvalidateInstructionCache(Block.base(),230 Block.allocatedSize());231}232 233SectionMemoryManager::~SectionMemoryManager() {234 for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {235 for (sys::MemoryBlock &Block : Group->AllocatedMem)236 MMapper->releaseMappedMemory(Block);237 }238}239 240SectionMemoryManager::MemoryMapper::~MemoryMapper() = default;241 242void SectionMemoryManager::anchor() {}243 244namespace {245// Trivial implementation of SectionMemoryManager::MemoryMapper that just calls246// into sys::Memory.247class DefaultMMapper final : public SectionMemoryManager::MemoryMapper {248public:249 sys::MemoryBlock250 allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose,251 size_t NumBytes, const sys::MemoryBlock *const NearBlock,252 unsigned Flags, std::error_code &EC) override {253 return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC);254 }255 256 std::error_code protectMappedMemory(const sys::MemoryBlock &Block,257 unsigned Flags) override {258 return sys::Memory::protectMappedMemory(Block, Flags);259 }260 261 std::error_code releaseMappedMemory(sys::MemoryBlock &M) override {262 return sys::Memory::releaseMappedMemory(M);263 }264};265} // namespace266 267SectionMemoryManager::SectionMemoryManager(MemoryMapper *UnownedMM)268 : MMapper(UnownedMM), OwnedMMapper(nullptr) {269 if (!MMapper) {270 OwnedMMapper = std::make_unique<DefaultMMapper>();271 MMapper = OwnedMMapper.get();272 }273}274 275} // namespace llvm276