390 lines · cpp
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file Implements MappedFileRegionArena.9///10/// A bump pointer allocator, backed by a memory-mapped file.11///12/// The effect we want is:13///14/// Step 1. If it doesn't exist, create the file with an initial size.15/// Step 2. Reserve virtual memory large enough for the max file size.16/// Step 3. Map the file into memory in the reserved region.17/// Step 4. Increase the file size and update the mapping when necessary.18///19/// However, updating the mapping is challenging when it needs to work portably,20/// and across multiple processes without locking for every read. Our current21/// implementation handles the steps above in following ways:22///23/// Step 1. Use \ref sys::fs::resize_file_sparse to grow the file to its max24/// size (typically several GB). If the file system doesn't support25/// sparse file, this may return a fully allocated file.26/// Step 2. Call \ref sys::fs::mapped_file_region to map the entire file.27/// Step 3. [Automatic as part of step 2.]28/// Step 4. If supported, use \c fallocate or similiar APIs to ensure the file29/// system storage for the sparse file so we won't end up with partial30/// file if the disk is out of space.31///32/// Additionally, we attempt to resize the file to its actual data size when33/// closing the mapping, if this is the only concurrent instance. This is done34/// using file locks. Shrinking the file mitigates problems with having large35/// files: on filesystems without sparse files it avoids unnecessary space use;36/// it also avoids allocating the full size if another process copies the file,37/// which typically loses sparseness. These mitigations only work while the file38/// is not in use.39///40/// The capacity and the header offset is determined by the first user of the41/// MappedFileRegionArena instance and any future mismatched value from the42/// original will result in error on creation.43///44/// To support resizing, we use two separate file locks:45/// 1. We use a shared reader lock on a ".shared" file until destruction.46/// 2. We use a lock on the main file during initialization - shared to check47/// the status, upgraded to exclusive to resize/initialize the file.48///49/// Then during destruction we attempt to get exclusive access on (1), which50/// requires no concurrent readers. If so, we shrink the file. Using two51/// separate locks simplifies the implementation and enables it to work on52/// platforms (e.g. Windows) where a shared/reader lock prevents writing.53//===----------------------------------------------------------------------===//54 55#include "llvm/CAS/MappedFileRegionArena.h"56#include "OnDiskCommon.h"57#include "llvm/ADT/StringExtras.h"58 59#if LLVM_ON_UNIX60#include <sys/stat.h>61#if __has_include(<sys/param.h>)62#include <sys/param.h>63#endif64#ifdef DEV_BSIZE65#define MAPPED_FILE_BSIZE DEV_BSIZE66#elif __linux__67#define MAPPED_FILE_BSIZE 51268#endif69#endif70 71using namespace llvm;72using namespace llvm::cas;73using namespace llvm::cas::ondisk;74 75namespace {76struct FileWithLock {77 std::string Path;78 int FD = -1;79 std::optional<sys::fs::LockKind> Locked;80 81private:82 FileWithLock(std::string PathStr, Error &E) : Path(std::move(PathStr)) {83 ErrorAsOutParameter EOP(&E);84 if (std::error_code EC = sys::fs::openFileForReadWrite(85 Path, FD, sys::fs::CD_OpenAlways, sys::fs::OF_None))86 E = createFileError(Path, EC);87 }88 89public:90 FileWithLock(FileWithLock &) = delete;91 FileWithLock(FileWithLock &&Other) {92 Path = std::move(Other.Path);93 FD = Other.FD;94 Other.FD = -1;95 Locked = Other.Locked;96 Other.Locked = std::nullopt;97 }98 99 ~FileWithLock() { consumeError(unlock()); }100 101 static Expected<FileWithLock> open(StringRef Path) {102 Error E = Error::success();103 FileWithLock Result(Path.str(), E);104 if (E)105 return std::move(E);106 return std::move(Result);107 }108 109 Error lock(sys::fs::LockKind LK) {110 assert(!Locked && "already locked");111 if (std::error_code EC = lockFileThreadSafe(FD, LK))112 return createFileError(Path, EC);113 Locked = LK;114 return Error::success();115 }116 117 Error switchLock(sys::fs::LockKind LK) {118 assert(Locked && "not locked");119 if (auto E = unlock())120 return E;121 122 return lock(LK);123 }124 125 Error unlock() {126 if (Locked) {127 Locked = std::nullopt;128 if (std::error_code EC = unlockFileThreadSafe(FD))129 return createFileError(Path, EC);130 }131 return Error::success();132 }133 134 // Return true if succeed to lock the file exclusively.135 bool tryLockExclusive() {136 assert(!Locked && "can only try to lock if not locked");137 if (tryLockFileThreadSafe(FD) == std::error_code()) {138 Locked = sys::fs::LockKind::Exclusive;139 return true;140 }141 142 return false;143 }144 145 // Release the lock so it will not be unlocked on destruction.146 void release() {147 Locked = std::nullopt;148 FD = -1;149 }150};151 152struct FileSizeInfo {153 uint64_t Size;154 uint64_t AllocatedSize;155 156 static ErrorOr<FileSizeInfo> get(sys::fs::file_t File);157};158} // end anonymous namespace159 160Expected<MappedFileRegionArena> MappedFileRegionArena::create(161 const Twine &Path, uint64_t Capacity, uint64_t HeaderOffset,162 function_ref<Error(MappedFileRegionArena &)> NewFileConstructor) {163 uint64_t MinCapacity = HeaderOffset + sizeof(Header);164 if (Capacity < MinCapacity)165 return createStringError(166 std::make_error_code(std::errc::invalid_argument),167 "capacity is too small to hold MappedFileRegionArena");168 169 MappedFileRegionArena Result;170 Result.Path = Path.str();171 172 // Open the shared lock file. See file comment for details of locking scheme.173 SmallString<128> SharedFilePath(Result.Path);174 SharedFilePath.append(".shared");175 176 auto SharedFileLock = FileWithLock::open(SharedFilePath);177 if (!SharedFileLock)178 return SharedFileLock.takeError();179 Result.SharedLockFD = SharedFileLock->FD;180 181 // Take shared/reader lock that will be held until destroyImpl if construction182 // is successful.183 if (auto E = SharedFileLock->lock(sys::fs::LockKind::Shared))184 return std::move(E);185 186 // Take shared/reader lock for initialization.187 auto MainFile = FileWithLock::open(Result.Path);188 if (!MainFile)189 return MainFile.takeError();190 if (Error E = MainFile->lock(sys::fs::LockKind::Shared))191 return std::move(E);192 Result.FD = MainFile->FD;193 194 sys::fs::file_t File = sys::fs::convertFDToNativeFile(MainFile->FD);195 auto FileSize = FileSizeInfo::get(File);196 if (!FileSize)197 return createFileError(Result.Path, FileSize.getError());198 199 // If the size is smaller than the capacity, we need to initialize the file.200 // It maybe empty, or may have been shrunk during a previous close.201 if (FileSize->Size < Capacity) {202 // Lock the file exclusively so only one process will do the initialization.203 if (Error E = MainFile->switchLock(sys::fs::LockKind::Exclusive))204 return std::move(E);205 // Retrieve the current size now that we have exclusive access.206 FileSize = FileSizeInfo::get(File);207 if (!FileSize)208 return createFileError(Result.Path, FileSize.getError());209 }210 211 if (FileSize->Size >= MinCapacity) {212 // File is initialized. Read out the header to check for capacity and213 // offset.214 SmallVector<char, sizeof(Header)> HeaderContent(sizeof(Header));215 auto Size = sys::fs::readNativeFileSlice(File, HeaderContent, HeaderOffset);216 if (!Size)217 return Size.takeError();218 219 Header H;220 memcpy(&H, HeaderContent.data(), sizeof(H));221 if (H.HeaderOffset != HeaderOffset)222 return createStringError(223 std::make_error_code(std::errc::invalid_argument),224 "specified header offset (" + utostr(HeaderOffset) +225 ") does not match existing config (" + utostr(H.HeaderOffset) +226 ")");227 228 // If the capacity doesn't match, use the existing capacity instead.229 if (H.Capacity != Capacity)230 Capacity = H.Capacity;231 }232 233 // If the size is smaller than capacity, we need to resize the file.234 if (FileSize->Size < Capacity) {235 assert(MainFile->Locked == sys::fs::LockKind::Exclusive);236 if (std::error_code EC =237 sys::fs::resize_file_sparse(MainFile->FD, Capacity))238 return createFileError(Result.Path, EC);239 }240 241 // Create the mapped region.242 {243 std::error_code EC;244 sys::fs::mapped_file_region Map(245 File, sys::fs::mapped_file_region::readwrite, Capacity, 0, EC);246 if (EC)247 return createFileError(Result.Path, EC);248 Result.Region = std::move(Map);249 }250 251 // Initialize the header.252 Result.initializeHeader(HeaderOffset);253 if (FileSize->Size < MinCapacity) {254 assert(MainFile->Locked == sys::fs::LockKind::Exclusive);255 // If we need to fully initialize the file, call NewFileConstructor.256 if (Error E = NewFileConstructor(Result))257 return std::move(E);258 259 Result.H->HeaderOffset.exchange(HeaderOffset);260 Result.H->Capacity.exchange(Capacity);261 }262 263 if (MainFile->Locked == sys::fs::LockKind::Exclusive) {264 // If holding an exclusive lock, we might have resized the file and265 // performed some read/write to the file. Query the file size again to make266 // sure everything is up-to-date. Otherwise, FileSize info is already267 // up-to-date.268 FileSize = FileSizeInfo::get(File);269 if (!FileSize)270 return createFileError(Result.Path, FileSize.getError());271 Result.H->AllocatedSize.exchange(FileSize->AllocatedSize);272 }273 274 // Release the shared lock so it can be closed in destoryImpl().275 SharedFileLock->release();276 return std::move(Result);277}278 279void MappedFileRegionArena::destroyImpl() {280 if (!FD)281 return;282 283 // Drop the shared lock indicating we are no longer accessing the file.284 if (SharedLockFD)285 (void)unlockFileThreadSafe(*SharedLockFD);286 287 // Attempt to truncate the file if we can get exclusive access. Ignore any288 // errors.289 if (H) {290 assert(SharedLockFD && "Must have shared lock file open");291 if (tryLockFileThreadSafe(*SharedLockFD) == std::error_code()) {292 size_t Size = size();293 // sync to file system to make sure all contents are up-to-date.294 (void)Region.sync();295 // unmap the file before resizing since that is the requirement for296 // some platforms.297 Region.unmap();298 (void)sys::fs::resize_file(*FD, Size);299 (void)unlockFileThreadSafe(*SharedLockFD);300 }301 }302 303 auto Close = [](std::optional<int> &FD) {304 if (FD) {305 sys::fs::file_t File = sys::fs::convertFDToNativeFile(*FD);306 sys::fs::closeFile(File);307 FD = std::nullopt;308 }309 };310 311 // Close the file and shared lock.312 Close(FD);313 Close(SharedLockFD);314}315 316void MappedFileRegionArena::initializeHeader(uint64_t HeaderOffset) {317 assert(capacity() < (uint64_t)INT64_MAX && "capacity must fit in int64_t");318 uint64_t HeaderEndOffset = HeaderOffset + sizeof(decltype(*H));319 assert(HeaderEndOffset <= capacity() &&320 "Expected end offset to be pre-allocated");321 assert(isAligned(Align::Of<decltype(*H)>(), HeaderOffset) &&322 "Expected end offset to be aligned");323 H = reinterpret_cast<decltype(H)>(data() + HeaderOffset);324 325 uint64_t ExistingValue = 0;326 if (!H->BumpPtr.compare_exchange_strong(ExistingValue, HeaderEndOffset))327 assert(ExistingValue >= HeaderEndOffset &&328 "Expected 0, or past the end of the header itself");329}330 331static Error createAllocatorOutOfSpaceError() {332 return createStringError(std::make_error_code(std::errc::not_enough_memory),333 "memory mapped file allocator is out of space");334}335 336Expected<int64_t> MappedFileRegionArena::allocateOffset(uint64_t AllocSize) {337 AllocSize = alignTo(AllocSize, getAlign());338 uint64_t OldEnd = H->BumpPtr.fetch_add(AllocSize);339 uint64_t NewEnd = OldEnd + AllocSize;340 if (LLVM_UNLIKELY(NewEnd > capacity())) {341 // Return the allocation. If the start already passed the end, that means342 // some other concurrent allocations already consumed all the capacity.343 // There is no need to return the original value. If the start was not344 // passed the end, current allocation certainly bumped it passed the end.345 // All other allocation afterwards must have failed and current allocation346 // is in charge of return the allocation back to a valid value.347 if (OldEnd <= capacity())348 (void)H->BumpPtr.exchange(OldEnd);349 350 return createAllocatorOutOfSpaceError();351 }352 353 uint64_t DiskSize = H->AllocatedSize;354 if (LLVM_UNLIKELY(NewEnd > DiskSize)) {355 uint64_t NewSize;356 // The minimum increment is a page, but allocate more to amortize the cost.357 constexpr uint64_t Increment = 1 * 1024 * 1024; // 1 MB358 if (Error E = preallocateFileTail(*FD, DiskSize, DiskSize + Increment)359 .moveInto(NewSize))360 return std::move(E);361 assert(NewSize >= DiskSize + Increment);362 // FIXME: on Darwin this can under-count the size if there is a race to363 // preallocate disk, because the semantics of F_PREALLOCATE are to add bytes364 // to the end of the file, not to allocate up to a fixed size.365 // Any discrepancy will be resolved the next time the file is truncated and366 // then reopend.367 while (DiskSize < NewSize)368 H->AllocatedSize.compare_exchange_strong(DiskSize, NewSize);369 }370 return OldEnd;371}372 373ErrorOr<FileSizeInfo> FileSizeInfo::get(sys::fs::file_t File) {374#if LLVM_ON_UNIX && defined(MAPPED_FILE_BSIZE)375 struct stat Status;376 int StatRet = ::fstat(File, &Status);377 if (StatRet)378 return errnoAsErrorCode();379 uint64_t AllocatedSize = uint64_t(Status.st_blksize) * MAPPED_FILE_BSIZE;380 return FileSizeInfo{uint64_t(Status.st_size), AllocatedSize};381#else382 // Fallback: assume the file is fully allocated. Note: this may result in383 // data loss on out-of-space.384 sys::fs::file_status Status;385 if (std::error_code EC = sys::fs::status(File, Status))386 return EC;387 return FileSizeInfo{Status.getSize(), Status.getSize()};388#endif389}390