371 lines · cpp
1//===---------- ExecutorSharedMemoryMapperService.cpp -----------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.h"10#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX11#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"12#include "llvm/Support/Process.h"13#include "llvm/Support/WindowsError.h"14#include <sstream>15 16#if defined(LLVM_ON_UNIX)17#include <errno.h>18#include <fcntl.h>19#include <sys/mman.h>20#if defined(__MVS__)21#include "llvm/Support/BLAKE3.h"22#include <sys/shm.h>23#endif24#include <unistd.h>25#endif26 27namespace llvm {28namespace orc {29namespace rt_bootstrap {30 31#if defined(_WIN32)32static DWORD getWindowsProtectionFlags(MemProt MP) {33 if (MP == MemProt::Read)34 return PAGE_READONLY;35 if (MP == MemProt::Write ||36 MP == (MemProt::Write | MemProt::Read)) {37 // Note: PAGE_WRITE is not supported by VirtualProtect38 return PAGE_READWRITE;39 }40 if (MP == (MemProt::Read | MemProt::Exec))41 return PAGE_EXECUTE_READ;42 if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec))43 return PAGE_EXECUTE_READWRITE;44 if (MP == MemProt::Exec)45 return PAGE_EXECUTE;46 47 return PAGE_NOACCESS;48}49#endif50 51Expected<std::pair<ExecutorAddr, std::string>>52ExecutorSharedMemoryMapperService::reserve(uint64_t Size) {53#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)54 55#if defined(LLVM_ON_UNIX)56 57 std::string SharedMemoryName;58 {59 std::stringstream SharedMemoryNameStream;60 SharedMemoryNameStream << "/jitlink_" << sys::Process::getProcessId() << '_'61 << (++SharedMemoryCount);62 SharedMemoryName = SharedMemoryNameStream.str();63 }64 65#if defined(__MVS__)66 ArrayRef<uint8_t> Data(67 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),68 SharedMemoryName.size());69 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);70 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());71 int SharedMemoryId =72 shmget(Key, Size, IPC_CREAT | IPC_EXCL | __IPC_SHAREAS | 0700);73 if (SharedMemoryId < 0)74 return errorCodeToError(errnoAsErrorCode());75 76 void *Addr = shmat(SharedMemoryId, nullptr, 0);77 if (Addr == reinterpret_cast<void *>(-1))78 return errorCodeToError(errnoAsErrorCode());79#else80 int SharedMemoryFile =81 shm_open(SharedMemoryName.c_str(), O_RDWR | O_CREAT | O_EXCL, 0700);82 if (SharedMemoryFile < 0)83 return errorCodeToError(errnoAsErrorCode());84 85 // by default size is 086 if (ftruncate(SharedMemoryFile, Size) < 0)87 return errorCodeToError(errnoAsErrorCode());88 89 void *Addr = mmap(nullptr, Size, PROT_NONE, MAP_SHARED, SharedMemoryFile, 0);90 if (Addr == MAP_FAILED)91 return errorCodeToError(errnoAsErrorCode());92 93 close(SharedMemoryFile);94#endif95 96#elif defined(_WIN32)97 98 std::string SharedMemoryName;99 {100 std::stringstream SharedMemoryNameStream;101 SharedMemoryNameStream << "jitlink_" << sys::Process::getProcessId() << '_'102 << (++SharedMemoryCount);103 SharedMemoryName = SharedMemoryNameStream.str();104 }105 106 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),107 SharedMemoryName.end());108 HANDLE SharedMemoryFile = CreateFileMappingW(109 INVALID_HANDLE_VALUE, NULL, PAGE_EXECUTE_READWRITE, Size >> 32,110 Size & 0xffffffff, WideSharedMemoryName.c_str());111 if (!SharedMemoryFile)112 return errorCodeToError(mapWindowsError(GetLastError()));113 114 void *Addr = MapViewOfFile(SharedMemoryFile,115 FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);116 if (!Addr) {117 CloseHandle(SharedMemoryFile);118 return errorCodeToError(mapWindowsError(GetLastError()));119 }120 121#endif122 123 {124 std::lock_guard<std::mutex> Lock(Mutex);125 Reservations[Addr].Size = Size;126#if defined(_WIN32)127 Reservations[Addr].SharedMemoryFile = SharedMemoryFile;128#endif129 }130 131 return std::make_pair(ExecutorAddr::fromPtr(Addr),132 std::move(SharedMemoryName));133#else134 return make_error<StringError>(135 "SharedMemoryMapper is not supported on this platform yet",136 inconvertibleErrorCode());137#endif138}139 140Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize(141 ExecutorAddr Reservation, tpctypes::SharedMemoryFinalizeRequest &FR) {142#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)143 144 ExecutorAddr MinAddr(~0ULL);145 146 // Contents are already in place147 for (auto &Segment : FR.Segments) {148 if (Segment.Addr < MinAddr)149 MinAddr = Segment.Addr;150 151#if defined(LLVM_ON_UNIX)152 153#if defined(__MVS__)154 // TODO Is it possible to change the protection level?155#else156 int NativeProt = 0;157 if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read)158 NativeProt |= PROT_READ;159 if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write)160 NativeProt |= PROT_WRITE;161 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)162 NativeProt |= PROT_EXEC;163 164 if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt))165 return errorCodeToError(errnoAsErrorCode());166#endif167 168#elif defined(_WIN32)169 170 DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot);171 172 if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt,173 &NativeProt))174 return errorCodeToError(mapWindowsError(GetLastError()));175 176#endif177 178 if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec)179 sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(),180 Segment.Size);181 }182 183 // Run finalization actions and get deinitlization action list.184 auto DeinitializeActions = shared::runFinalizeActions(FR.Actions);185 if (!DeinitializeActions) {186 return DeinitializeActions.takeError();187 }188 189 {190 std::lock_guard<std::mutex> Lock(Mutex);191 Allocations[MinAddr].DeinitializationActions =192 std::move(*DeinitializeActions);193 Reservations[Reservation.toPtr<void *>()].Allocations.push_back(MinAddr);194 }195 196 return MinAddr;197 198#else199 return make_error<StringError>(200 "SharedMemoryMapper is not supported on this platform yet",201 inconvertibleErrorCode());202#endif203}204 205Error ExecutorSharedMemoryMapperService::deinitialize(206 const std::vector<ExecutorAddr> &Bases) {207 Error AllErr = Error::success();208 209 {210 std::lock_guard<std::mutex> Lock(Mutex);211 212 for (auto Base : llvm::reverse(Bases)) {213 if (Error Err = shared::runDeallocActions(214 Allocations[Base].DeinitializationActions)) {215 AllErr = joinErrors(std::move(AllErr), std::move(Err));216 }217 218 // Remove the allocation from the allocation list of its reservation219 for (auto &Reservation : Reservations) {220 auto AllocationIt = llvm::find(Reservation.second.Allocations, Base);221 if (AllocationIt != Reservation.second.Allocations.end()) {222 Reservation.second.Allocations.erase(AllocationIt);223 break;224 }225 }226 227 Allocations.erase(Base);228 }229 }230 231 return AllErr;232}233 234Error ExecutorSharedMemoryMapperService::release(235 const std::vector<ExecutorAddr> &Bases) {236#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)237 Error Err = Error::success();238 239 for (auto Base : Bases) {240 std::vector<ExecutorAddr> AllocAddrs;241 size_t Size;242 243#if defined(_WIN32)244 HANDLE SharedMemoryFile;245#endif246 247 {248 std::lock_guard<std::mutex> Lock(Mutex);249 auto &R = Reservations[Base.toPtr<void *>()];250 Size = R.Size;251 252#if defined(_WIN32)253 SharedMemoryFile = R.SharedMemoryFile;254#endif255 256 AllocAddrs.swap(R.Allocations);257 }258 259 // deinitialize sub allocations260 if (Error E = deinitialize(AllocAddrs))261 Err = joinErrors(std::move(Err), std::move(E));262 263#if defined(LLVM_ON_UNIX)264 265#if defined(__MVS__)266 (void)Size;267 268 if (shmdt(Base.toPtr<void *>()) < 0)269 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));270#else271 if (munmap(Base.toPtr<void *>(), Size) != 0)272 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));273#endif274 275#elif defined(_WIN32)276 (void)Size;277 278 if (!UnmapViewOfFile(Base.toPtr<void *>()))279 Err = joinErrors(std::move(Err),280 errorCodeToError(mapWindowsError(GetLastError())));281 282 CloseHandle(SharedMemoryFile);283 284#endif285 286 std::lock_guard<std::mutex> Lock(Mutex);287 Reservations.erase(Base.toPtr<void *>());288 }289 290 return Err;291#else292 return make_error<StringError>(293 "SharedMemoryMapper is not supported on this platform yet",294 inconvertibleErrorCode());295#endif296}297 298Error ExecutorSharedMemoryMapperService::shutdown() {299 if (Reservations.empty())300 return Error::success();301 302 std::vector<ExecutorAddr> ReservationAddrs;303 ReservationAddrs.reserve(Reservations.size());304 for (const auto &R : Reservations)305 ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));306 307 return release(std::move(ReservationAddrs));308}309 310void ExecutorSharedMemoryMapperService::addBootstrapSymbols(311 StringMap<ExecutorAddr> &M) {312 M[rt::ExecutorSharedMemoryMapperServiceInstanceName] =313 ExecutorAddr::fromPtr(this);314 M[rt::ExecutorSharedMemoryMapperServiceReserveWrapperName] =315 ExecutorAddr::fromPtr(&reserveWrapper);316 M[rt::ExecutorSharedMemoryMapperServiceInitializeWrapperName] =317 ExecutorAddr::fromPtr(&initializeWrapper);318 M[rt::ExecutorSharedMemoryMapperServiceDeinitializeWrapperName] =319 ExecutorAddr::fromPtr(&deinitializeWrapper);320 M[rt::ExecutorSharedMemoryMapperServiceReleaseWrapperName] =321 ExecutorAddr::fromPtr(&releaseWrapper);322}323 324llvm::orc::shared::CWrapperFunctionResult325ExecutorSharedMemoryMapperService::reserveWrapper(const char *ArgData,326 size_t ArgSize) {327 return shared::WrapperFunction<328 rt::SPSExecutorSharedMemoryMapperServiceReserveSignature>::329 handle(ArgData, ArgSize,330 shared::makeMethodWrapperHandler(331 &ExecutorSharedMemoryMapperService::reserve))332 .release();333}334 335llvm::orc::shared::CWrapperFunctionResult336ExecutorSharedMemoryMapperService::initializeWrapper(const char *ArgData,337 size_t ArgSize) {338 return shared::WrapperFunction<339 rt::SPSExecutorSharedMemoryMapperServiceInitializeSignature>::340 handle(ArgData, ArgSize,341 shared::makeMethodWrapperHandler(342 &ExecutorSharedMemoryMapperService::initialize))343 .release();344}345 346llvm::orc::shared::CWrapperFunctionResult347ExecutorSharedMemoryMapperService::deinitializeWrapper(const char *ArgData,348 size_t ArgSize) {349 return shared::WrapperFunction<350 rt::SPSExecutorSharedMemoryMapperServiceDeinitializeSignature>::351 handle(ArgData, ArgSize,352 shared::makeMethodWrapperHandler(353 &ExecutorSharedMemoryMapperService::deinitialize))354 .release();355}356 357llvm::orc::shared::CWrapperFunctionResult358ExecutorSharedMemoryMapperService::releaseWrapper(const char *ArgData,359 size_t ArgSize) {360 return shared::WrapperFunction<361 rt::SPSExecutorSharedMemoryMapperServiceReleaseSignature>::362 handle(ArgData, ArgSize,363 shared::makeMethodWrapperHandler(364 &ExecutorSharedMemoryMapperService::release))365 .release();366}367 368} // namespace rt_bootstrap369} // end namespace orc370} // end namespace llvm371