600 lines · cpp
1//===--- MemoryBuffer.cpp - Memory Buffer implementation ------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the MemoryBuffer interface.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/Support/MemoryBuffer.h"14#include "llvm/ADT/STLExtras.h"15#include "llvm/ADT/SmallString.h"16#include "llvm/Config/config.h"17#include "llvm/Support/Alignment.h"18#include "llvm/Support/AutoConvert.h"19#include "llvm/Support/Errc.h"20#include "llvm/Support/Error.h"21#include "llvm/Support/ErrorHandling.h"22#include "llvm/Support/FileSystem.h"23#include "llvm/Support/Process.h"24#include "llvm/Support/Program.h"25#include "llvm/Support/SmallVectorMemoryBuffer.h"26#include <algorithm>27#include <cassert>28#include <cstring>29#include <new>30#include <sys/types.h>31#include <system_error>32#if !defined(_MSC_VER) && !defined(__MINGW32__)33#include <unistd.h>34#else35#include <io.h>36#endif37 38using namespace llvm;39 40//===----------------------------------------------------------------------===//41// MemoryBuffer implementation itself.42//===----------------------------------------------------------------------===//43 44MemoryBuffer::~MemoryBuffer() = default;45 46/// init - Initialize this MemoryBuffer as a reference to externally allocated47/// memory, memory that we know is already null terminated.48void MemoryBuffer::init(const char *BufStart, const char *BufEnd,49 bool RequiresNullTerminator) {50 assert((!RequiresNullTerminator || BufEnd[0] == 0) &&51 "Buffer is not null terminated!");52 BufferStart = BufStart;53 BufferEnd = BufEnd;54}55 56//===----------------------------------------------------------------------===//57// MemoryBufferMem implementation.58//===----------------------------------------------------------------------===//59 60/// CopyStringRef - Copies contents of a StringRef into a block of memory and61/// null-terminates it.62static void CopyStringRef(char *Memory, StringRef Data) {63 if (!Data.empty())64 memcpy(Memory, Data.data(), Data.size());65 Memory[Data.size()] = 0; // Null terminate string.66}67 68namespace {69struct NamedBufferAlloc {70 const Twine &Name;71 NamedBufferAlloc(const Twine &Name) : Name(Name) {}72};73} // namespace74 75void *operator new(size_t N, const NamedBufferAlloc &Alloc) {76 SmallString<256> NameBuf;77 StringRef NameRef = Alloc.Name.toStringRef(NameBuf);78 79 // We use malloc() and manually handle it returning null instead of calling80 // operator new because we need all uses of NamedBufferAlloc to be81 // deallocated with a call to free() due to needing to use malloc() in82 // WritableMemoryBuffer::getNewUninitMemBuffer() to work around the out-of-83 // memory handler installed by default in LLVM. See operator delete() member84 // functions within this file for the paired call to free().85 char *Mem =86 static_cast<char *>(std::malloc(N + sizeof(size_t) + NameRef.size() + 1));87 if (!Mem)88 llvm::report_bad_alloc_error("Allocation failed");89 *reinterpret_cast<size_t *>(Mem + N) = NameRef.size();90 CopyStringRef(Mem + N + sizeof(size_t), NameRef);91 return Mem;92}93 94namespace {95/// MemoryBufferMem - Named MemoryBuffer pointing to a block of memory.96template<typename MB>97class MemoryBufferMem : public MB {98public:99 MemoryBufferMem(StringRef InputData, bool RequiresNullTerminator) {100 MemoryBuffer::init(InputData.begin(), InputData.end(),101 RequiresNullTerminator);102 }103 104 /// Disable sized deallocation for MemoryBufferMem, because it has105 /// tail-allocated data.106 void operator delete(void *p) { std::free(p); }107 108 StringRef getBufferIdentifier() const override {109 // The name is stored after the class itself.110 return StringRef(reinterpret_cast<const char *>(this + 1) + sizeof(size_t),111 *reinterpret_cast<const size_t *>(this + 1));112 }113 114 MemoryBuffer::BufferKind getBufferKind() const override {115 return MemoryBuffer::MemoryBuffer_Malloc;116 }117};118} // namespace119 120template <typename MB>121static ErrorOr<std::unique_ptr<MB>>122getFileAux(const Twine &Filename, uint64_t MapSize, uint64_t Offset,123 bool IsText, bool RequiresNullTerminator, bool IsVolatile,124 std::optional<Align> Alignment);125 126std::unique_ptr<MemoryBuffer>127MemoryBuffer::getMemBuffer(StringRef InputData, StringRef BufferName,128 bool RequiresNullTerminator) {129 auto *Ret = new (NamedBufferAlloc(BufferName))130 MemoryBufferMem<MemoryBuffer>(InputData, RequiresNullTerminator);131 return std::unique_ptr<MemoryBuffer>(Ret);132}133 134std::unique_ptr<MemoryBuffer>135MemoryBuffer::getMemBuffer(MemoryBufferRef Ref, bool RequiresNullTerminator) {136 return std::unique_ptr<MemoryBuffer>(getMemBuffer(137 Ref.getBuffer(), Ref.getBufferIdentifier(), RequiresNullTerminator));138}139 140static ErrorOr<std::unique_ptr<WritableMemoryBuffer>>141getMemBufferCopyImpl(StringRef InputData, const Twine &BufferName) {142 auto Buf =143 WritableMemoryBuffer::getNewUninitMemBuffer(InputData.size(), BufferName);144 if (!Buf)145 return make_error_code(errc::not_enough_memory);146 // Calling memcpy with null src/dst is UB, and an empty StringRef is147 // represented with {nullptr, 0}.148 llvm::copy(InputData, Buf->getBufferStart());149 return std::move(Buf);150}151 152std::unique_ptr<MemoryBuffer>153MemoryBuffer::getMemBufferCopy(StringRef InputData, const Twine &BufferName) {154 auto Buf = getMemBufferCopyImpl(InputData, BufferName);155 if (Buf)156 return std::move(*Buf);157 return nullptr;158}159 160ErrorOr<std::unique_ptr<MemoryBuffer>>161MemoryBuffer::getFileOrSTDIN(const Twine &Filename, bool IsText,162 bool RequiresNullTerminator,163 std::optional<Align> Alignment) {164 SmallString<256> NameBuf;165 StringRef NameRef = Filename.toStringRef(NameBuf);166 167 if (NameRef == "-")168 return getSTDIN();169 return getFile(Filename, IsText, RequiresNullTerminator,170 /*IsVolatile=*/false, Alignment);171}172 173ErrorOr<std::unique_ptr<MemoryBuffer>>174MemoryBuffer::getFileSlice(const Twine &FilePath, uint64_t MapSize,175 uint64_t Offset, bool IsVolatile,176 std::optional<Align> Alignment) {177 return getFileAux<MemoryBuffer>(FilePath, MapSize, Offset, /*IsText=*/false,178 /*RequiresNullTerminator=*/false, IsVolatile,179 Alignment);180}181 182//===----------------------------------------------------------------------===//183// MemoryBuffer::getFile implementation.184//===----------------------------------------------------------------------===//185 186namespace {187 188template <typename MB>189constexpr sys::fs::mapped_file_region::mapmode Mapmode =190 sys::fs::mapped_file_region::readonly;191template <>192constexpr sys::fs::mapped_file_region::mapmode Mapmode<MemoryBuffer> =193 sys::fs::mapped_file_region::readonly;194template <>195constexpr sys::fs::mapped_file_region::mapmode Mapmode<WritableMemoryBuffer> =196 sys::fs::mapped_file_region::priv;197template <>198constexpr sys::fs::mapped_file_region::mapmode199 Mapmode<WriteThroughMemoryBuffer> = sys::fs::mapped_file_region::readwrite;200 201/// Memory maps a file descriptor using sys::fs::mapped_file_region.202///203/// This handles converting the offset into a legal offset on the platform.204template<typename MB>205class MemoryBufferMMapFile : public MB {206 sys::fs::mapped_file_region MFR;207 208 static uint64_t getLegalMapOffset(uint64_t Offset) {209 return Offset & ~(sys::fs::mapped_file_region::alignment() - 1);210 }211 212 static uint64_t getLegalMapSize(uint64_t Len, uint64_t Offset) {213 return Len + (Offset - getLegalMapOffset(Offset));214 }215 216 const char *getStart(uint64_t Len, uint64_t Offset) {217 return MFR.const_data() + (Offset - getLegalMapOffset(Offset));218 }219 220public:221 MemoryBufferMMapFile(bool RequiresNullTerminator, sys::fs::file_t FD, uint64_t Len,222 uint64_t Offset, std::error_code &EC)223 : MFR(FD, Mapmode<MB>, getLegalMapSize(Len, Offset),224 getLegalMapOffset(Offset), EC) {225 if (!EC) {226 const char *Start = getStart(Len, Offset);227 MemoryBuffer::init(Start, Start + Len, RequiresNullTerminator);228 }229 }230 231 /// Disable sized deallocation for MemoryBufferMMapFile, because it has232 /// tail-allocated data.233 void operator delete(void *p) { std::free(p); }234 235 StringRef getBufferIdentifier() const override {236 // The name is stored after the class itself.237 return StringRef(reinterpret_cast<const char *>(this + 1) + sizeof(size_t),238 *reinterpret_cast<const size_t *>(this + 1));239 }240 241 MemoryBuffer::BufferKind getBufferKind() const override {242 return MemoryBuffer::MemoryBuffer_MMap;243 }244 245 void dontNeedIfMmap() override { MFR.dontNeed(); }246};247} // namespace248 249static ErrorOr<std::unique_ptr<WritableMemoryBuffer>>250getMemoryBufferForStream(sys::fs::file_t FD, const Twine &BufferName) {251 SmallString<sys::fs::DefaultReadChunkSize> Buffer;252 if (Error E = sys::fs::readNativeFileToEOF(FD, Buffer))253 return errorToErrorCode(std::move(E));254 return getMemBufferCopyImpl(Buffer, BufferName);255}256 257ErrorOr<std::unique_ptr<MemoryBuffer>>258MemoryBuffer::getFile(const Twine &Filename, bool IsText,259 bool RequiresNullTerminator, bool IsVolatile,260 std::optional<Align> Alignment) {261 return getFileAux<MemoryBuffer>(Filename, /*MapSize=*/-1, /*Offset=*/0,262 IsText, RequiresNullTerminator, IsVolatile,263 Alignment);264}265 266template <typename MB>267static ErrorOr<std::unique_ptr<MB>>268getOpenFileImpl(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize,269 uint64_t MapSize, int64_t Offset, bool RequiresNullTerminator,270 bool IsVolatile, std::optional<Align> Alignment);271 272template <typename MB>273static ErrorOr<std::unique_ptr<MB>>274getFileAux(const Twine &Filename, uint64_t MapSize, uint64_t Offset,275 bool IsText, bool RequiresNullTerminator, bool IsVolatile,276 std::optional<Align> Alignment) {277 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(278 Filename, IsText ? sys::fs::OF_TextWithCRLF : sys::fs::OF_None);279 if (!FDOrErr)280 return errorToErrorCode(FDOrErr.takeError());281 sys::fs::file_t FD = *FDOrErr;282 auto Ret = getOpenFileImpl<MB>(FD, Filename, /*FileSize=*/-1, MapSize, Offset,283 RequiresNullTerminator, IsVolatile, Alignment);284 sys::fs::closeFile(FD);285 return Ret;286}287 288ErrorOr<std::unique_ptr<WritableMemoryBuffer>>289WritableMemoryBuffer::getFile(const Twine &Filename, bool IsVolatile,290 std::optional<Align> Alignment) {291 return getFileAux<WritableMemoryBuffer>(292 Filename, /*MapSize=*/-1, /*Offset=*/0, /*IsText=*/false,293 /*RequiresNullTerminator=*/false, IsVolatile, Alignment);294}295 296ErrorOr<std::unique_ptr<WritableMemoryBuffer>>297WritableMemoryBuffer::getFileSlice(const Twine &Filename, uint64_t MapSize,298 uint64_t Offset, bool IsVolatile,299 std::optional<Align> Alignment) {300 return getFileAux<WritableMemoryBuffer>(301 Filename, MapSize, Offset, /*IsText=*/false,302 /*RequiresNullTerminator=*/false, IsVolatile, Alignment);303}304 305std::unique_ptr<WritableMemoryBuffer>306WritableMemoryBuffer::getNewUninitMemBuffer(size_t Size,307 const Twine &BufferName,308 std::optional<Align> Alignment) {309 using MemBuffer = MemoryBufferMem<WritableMemoryBuffer>;310 311 // Use 16-byte alignment if no alignment is specified.312 Align BufAlign = Alignment.value_or(Align(16));313 314 // Allocate space for the MemoryBuffer, the data and the name. It is important315 // that MemoryBuffer and data are aligned so PointerIntPair works with them.316 SmallString<256> NameBuf;317 StringRef NameRef = BufferName.toStringRef(NameBuf);318 319 size_t StringLen = sizeof(MemBuffer) + sizeof(size_t) + NameRef.size() + 1;320 size_t RealLen = StringLen + Size + 1 + BufAlign.value();321 if (RealLen <= Size) // Check for rollover.322 return nullptr;323 // We use a call to malloc() rather than a call to a non-throwing operator324 // new() because LLVM unconditionally installs an out of memory new handler325 // when exceptions are disabled. This new handler intentionally crashes to326 // aid with debugging, but that makes non-throwing new calls unhelpful.327 // See MemoryBufferMem::operator delete() for the paired call to free(), and328 // llvm::install_out_of_memory_new_handler() for the installation of the329 // custom new handler.330 char *Mem = static_cast<char *>(std::malloc(RealLen));331 if (!Mem)332 return nullptr;333 334 // The name is stored after the class itself.335 *reinterpret_cast<size_t *>(Mem + sizeof(MemBuffer)) = NameRef.size();336 CopyStringRef(Mem + sizeof(MemBuffer) + sizeof(size_t), NameRef);337 338 // The buffer begins after the name and must be aligned.339 char *Buf = (char *)alignAddr(Mem + StringLen, BufAlign);340 Buf[Size] = 0; // Null terminate buffer.341 342 auto *Ret = new (Mem) MemBuffer(StringRef(Buf, Size), true);343 return std::unique_ptr<WritableMemoryBuffer>(Ret);344}345 346std::unique_ptr<WritableMemoryBuffer>347WritableMemoryBuffer::getNewMemBuffer(size_t Size, const Twine &BufferName) {348 auto SB = WritableMemoryBuffer::getNewUninitMemBuffer(Size, BufferName);349 if (!SB)350 return nullptr;351 memset(SB->getBufferStart(), 0, Size);352 return SB;353}354 355static bool shouldUseMmap(sys::fs::file_t FD,356 size_t FileSize,357 size_t MapSize,358 off_t Offset,359 bool RequiresNullTerminator,360 int PageSize,361 bool IsVolatile) {362#if defined(__MVS__)363 // zOS Enhanced ASCII auto convert does not support mmap.364 return false;365#endif366 367 // mmap may leave the buffer without null terminator if the file size changed368 // by the time the last page is mapped in, so avoid it if the file size is369 // likely to change.370 if (IsVolatile && RequiresNullTerminator)371 return false;372 373 // We don't use mmap for small files because this can severely fragment our374 // address space.375 if (MapSize < 4 * 4096 || MapSize < (unsigned)PageSize)376 return false;377 378 if (!RequiresNullTerminator)379 return true;380 381 // If we don't know the file size, use fstat to find out. fstat on an open382 // file descriptor is cheaper than stat on a random path.383 // FIXME: this chunk of code is duplicated, but it avoids a fstat when384 // RequiresNullTerminator = false and MapSize != -1.385 if (FileSize == size_t(-1)) {386 sys::fs::file_status Status;387 if (sys::fs::status(FD, Status))388 return false;389 FileSize = Status.getSize();390 }391 392 // If we need a null terminator and the end of the map is inside the file,393 // we cannot use mmap.394 size_t End = Offset + MapSize;395 assert(End <= FileSize);396 if (End != FileSize)397 return false;398 399 // Don't try to map files that are exactly a multiple of the system page size400 // if we need a null terminator.401 if ((FileSize & (PageSize -1)) == 0)402 return false;403 404#if defined(__CYGWIN__)405 // Don't try to map files that are exactly a multiple of the physical page size406 // if we need a null terminator.407 // FIXME: We should reorganize again getPageSize() on Win32.408 if ((FileSize & (4096 - 1)) == 0)409 return false;410#endif411 412 return true;413}414 415static ErrorOr<std::unique_ptr<WriteThroughMemoryBuffer>>416getReadWriteFile(const Twine &Filename, uint64_t FileSize, uint64_t MapSize,417 uint64_t Offset) {418 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForReadWrite(419 Filename, sys::fs::CD_OpenExisting, sys::fs::OF_None);420 if (!FDOrErr)421 return errorToErrorCode(FDOrErr.takeError());422 sys::fs::file_t FD = *FDOrErr;423 424 // Default is to map the full file.425 if (MapSize == uint64_t(-1)) {426 // If we don't know the file size, use fstat to find out. fstat on an open427 // file descriptor is cheaper than stat on a random path.428 if (FileSize == uint64_t(-1)) {429 sys::fs::file_status Status;430 std::error_code EC = sys::fs::status(FD, Status);431 if (EC)432 return EC;433 434 // If this not a file or a block device (e.g. it's a named pipe435 // or character device), we can't mmap it, so error out.436 sys::fs::file_type Type = Status.type();437 if (Type != sys::fs::file_type::regular_file &&438 Type != sys::fs::file_type::block_file)439 return make_error_code(errc::invalid_argument);440 441 FileSize = Status.getSize();442 }443 MapSize = FileSize;444 }445 446 std::error_code EC;447 std::unique_ptr<WriteThroughMemoryBuffer> Result(448 new (NamedBufferAlloc(Filename))449 MemoryBufferMMapFile<WriteThroughMemoryBuffer>(false, FD, MapSize,450 Offset, EC));451 if (EC)452 return EC;453 return std::move(Result);454}455 456ErrorOr<std::unique_ptr<WriteThroughMemoryBuffer>>457WriteThroughMemoryBuffer::getFile(const Twine &Filename, int64_t FileSize) {458 return getReadWriteFile(Filename, FileSize, FileSize, 0);459}460 461/// Map a subrange of the specified file as a WritableMemoryBuffer.462ErrorOr<std::unique_ptr<WriteThroughMemoryBuffer>>463WriteThroughMemoryBuffer::getFileSlice(const Twine &Filename, uint64_t MapSize,464 uint64_t Offset) {465 return getReadWriteFile(Filename, -1, MapSize, Offset);466}467 468template <typename MB>469static ErrorOr<std::unique_ptr<MB>>470getOpenFileImpl(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize,471 uint64_t MapSize, int64_t Offset, bool RequiresNullTerminator,472 bool IsVolatile, std::optional<Align> Alignment) {473 static int PageSize = sys::Process::getPageSizeEstimate();474 475 // Default is to map the full file.476 if (MapSize == uint64_t(-1)) {477 // If we don't know the file size, use fstat to find out. fstat on an open478 // file descriptor is cheaper than stat on a random path.479 if (FileSize == uint64_t(-1)) {480 sys::fs::file_status Status;481 std::error_code EC = sys::fs::status(FD, Status);482 if (EC)483 return EC;484 485 // If this not a file or a block device (e.g. it's a named pipe486 // or character device), we can't trust the size. Create the memory487 // buffer by copying off the stream.488 sys::fs::file_type Type = Status.type();489 if (Type != sys::fs::file_type::regular_file &&490 Type != sys::fs::file_type::block_file)491 return getMemoryBufferForStream(FD, Filename);492 493 FileSize = Status.getSize();494 }495 MapSize = FileSize;496 }497 498 if (shouldUseMmap(FD, FileSize, MapSize, Offset, RequiresNullTerminator,499 PageSize, IsVolatile)) {500 std::error_code EC;501 std::unique_ptr<MB> Result(502 new (NamedBufferAlloc(Filename)) MemoryBufferMMapFile<MB>(503 RequiresNullTerminator, FD, MapSize, Offset, EC));504 if (!EC) {505 // On at least Linux, and possibly on other systems, mmap may return pages506 // from the page cache that are not properly filled with trailing zeroes,507 // if some prior user of the page wrote non-zero bytes. Detect this and508 // don't use mmap in that case.509 if (!RequiresNullTerminator || *Result->getBufferEnd() == '\0')510 return std::move(Result);511 }512 }513 514#ifdef __MVS__515 ErrorOr<bool> NeedsConversion = needConversion(Filename, FD);516 if (std::error_code EC = NeedsConversion.getError())517 return EC;518 // File size may increase due to EBCDIC -> UTF-8 conversion, therefore we519 // cannot trust the file size and we create the memory buffer by copying520 // off the stream.521 // Note: This only works with the assumption of reading a full file (i.e,522 // Offset == 0 and MapSize == FileSize). Reading a file slice does not work.523 if (*NeedsConversion && Offset == 0 && MapSize == FileSize)524 return getMemoryBufferForStream(FD, Filename);525#endif526 527 auto Buf =528 WritableMemoryBuffer::getNewUninitMemBuffer(MapSize, Filename, Alignment);529 if (!Buf) {530 // Failed to create a buffer. The only way it can fail is if531 // new(std::nothrow) returns 0.532 return make_error_code(errc::not_enough_memory);533 }534 535 // Read until EOF, zero-initialize the rest.536 MutableArrayRef<char> ToRead = Buf->getBuffer();537 while (!ToRead.empty()) {538 Expected<size_t> ReadBytes =539 sys::fs::readNativeFileSlice(FD, ToRead, Offset);540 if (!ReadBytes)541 return errorToErrorCode(ReadBytes.takeError());542 if (*ReadBytes == 0) {543 std::memset(ToRead.data(), 0, ToRead.size());544 break;545 }546 ToRead = ToRead.drop_front(*ReadBytes);547 Offset += *ReadBytes;548 }549 550 return std::move(Buf);551}552 553ErrorOr<std::unique_ptr<MemoryBuffer>>554MemoryBuffer::getOpenFile(sys::fs::file_t FD, const Twine &Filename,555 uint64_t FileSize, bool RequiresNullTerminator,556 bool IsVolatile, std::optional<Align> Alignment) {557 return getOpenFileImpl<MemoryBuffer>(FD, Filename, FileSize, FileSize, 0,558 RequiresNullTerminator, IsVolatile,559 Alignment);560}561 562ErrorOr<std::unique_ptr<MemoryBuffer>> MemoryBuffer::getOpenFileSlice(563 sys::fs::file_t FD, const Twine &Filename, uint64_t MapSize, int64_t Offset,564 bool IsVolatile, std::optional<Align> Alignment) {565 assert(MapSize != uint64_t(-1));566 return getOpenFileImpl<MemoryBuffer>(FD, Filename, -1, MapSize, Offset, false,567 IsVolatile, Alignment);568}569 570ErrorOr<std::unique_ptr<MemoryBuffer>> MemoryBuffer::getSTDIN() {571 // Read in all of the data from stdin, we cannot mmap stdin.572 //573 // FIXME: That isn't necessarily true, we should try to mmap stdin and574 // fallback if it fails.575 sys::ChangeStdinMode(sys::fs::OF_Text);576 577 return getMemoryBufferForStream(sys::fs::getStdinHandle(), "<stdin>");578}579 580ErrorOr<std::unique_ptr<MemoryBuffer>>581MemoryBuffer::getFileAsStream(const Twine &Filename) {582 Expected<sys::fs::file_t> FDOrErr =583 sys::fs::openNativeFileForRead(Filename, sys::fs::OF_None);584 if (!FDOrErr)585 return errorToErrorCode(FDOrErr.takeError());586 sys::fs::file_t FD = *FDOrErr;587 ErrorOr<std::unique_ptr<MemoryBuffer>> Ret =588 getMemoryBufferForStream(FD, Filename);589 sys::fs::closeFile(FD);590 return Ret;591}592 593MemoryBufferRef MemoryBuffer::getMemBufferRef() const {594 StringRef Data = getBuffer();595 StringRef Identifier = getBufferIdentifier();596 return MemoryBufferRef(Data, Identifier);597}598 599SmallVectorMemoryBuffer::~SmallVectorMemoryBuffer() = default;600