488 lines · cpp
1//===-- msan_allocator.cpp -------------------------- ---------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a part of MemorySanitizer.10//11// MemorySanitizer allocator.12//===----------------------------------------------------------------------===//13 14#include "msan_allocator.h"15 16#include "msan.h"17#include "msan_interface_internal.h"18#include "msan_origin.h"19#include "msan_poisoning.h"20#include "msan_thread.h"21#include "sanitizer_common/sanitizer_allocator.h"22#include "sanitizer_common/sanitizer_allocator_checks.h"23#include "sanitizer_common/sanitizer_allocator_interface.h"24#include "sanitizer_common/sanitizer_allocator_report.h"25#include "sanitizer_common/sanitizer_errno.h"26 27using namespace __msan;28 29namespace {30struct Metadata {31 uptr requested_size;32};33 34struct MsanMapUnmapCallback {35 void OnMap(uptr p, uptr size) const {}36 void OnMapSecondary(uptr p, uptr size, uptr user_begin,37 uptr user_size) const {}38 void OnUnmap(uptr p, uptr size) const {39 __msan_unpoison((void *)p, size);40 41 // We are about to unmap a chunk of user memory.42 // Mark the corresponding shadow memory as not needed.43 uptr shadow_p = MEM_TO_SHADOW(p);44 ReleaseMemoryPagesToOS(shadow_p, shadow_p + size);45 if (__msan_get_track_origins()) {46 uptr origin_p = MEM_TO_ORIGIN(p);47 ReleaseMemoryPagesToOS(origin_p, origin_p + size);48 }49 }50};51 52// Note: to ensure that the allocator is compatible with the application memory53// layout (especially with high-entropy ASLR), kSpaceBeg and kSpaceSize must be54// duplicated as MappingDesc::ALLOCATOR in msan.h.55#if defined(__mips64)56const uptr kMaxAllowedMallocSize = 2UL << 30;57 58struct AP32 {59 static const uptr kSpaceBeg = SANITIZER_MMAP_BEGIN;60 static const u64 kSpaceSize = SANITIZER_MMAP_RANGE_SIZE;61 static const uptr kMetadataSize = sizeof(Metadata);62 using SizeClassMap = __sanitizer::CompactSizeClassMap;63 static const uptr kRegionSizeLog = 20;64 using AddressSpaceView = LocalAddressSpaceView;65 using MapUnmapCallback = MsanMapUnmapCallback;66 static const uptr kFlags = 0;67};68using PrimaryAllocator = SizeClassAllocator32<AP32>;69#elif defined(__x86_64__)70#if SANITIZER_NETBSD || SANITIZER_LINUX71const uptr kAllocatorSpace = 0x700000000000ULL;72#else73const uptr kAllocatorSpace = 0x600000000000ULL;74#endif75const uptr kMaxAllowedMallocSize = 1ULL << 40;76 77struct AP64 { // Allocator64 parameters. Deliberately using a short name.78 static const uptr kSpaceBeg = kAllocatorSpace;79 static const uptr kSpaceSize = 0x40000000000; // 4T.80 static const uptr kMetadataSize = sizeof(Metadata);81 using SizeClassMap = DefaultSizeClassMap;82 using MapUnmapCallback = MsanMapUnmapCallback;83 static const uptr kFlags = 0;84 using AddressSpaceView = LocalAddressSpaceView;85};86 87using PrimaryAllocator = SizeClassAllocator64<AP64>;88 89#elif defined(__loongarch_lp64)90const uptr kAllocatorSpace = 0x700000000000ULL;91const uptr kMaxAllowedMallocSize = 8UL << 30;92 93struct AP64 { // Allocator64 parameters. Deliberately using a short name.94 static const uptr kSpaceBeg = kAllocatorSpace;95 static const uptr kSpaceSize = 0x40000000000; // 4T.96 static const uptr kMetadataSize = sizeof(Metadata);97 using SizeClassMap = DefaultSizeClassMap;98 using MapUnmapCallback = MsanMapUnmapCallback;99 static const uptr kFlags = 0;100 using AddressSpaceView = LocalAddressSpaceView;101};102 103using PrimaryAllocator = SizeClassAllocator64<AP64>;104 105#elif defined(__powerpc64__)106const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G107 108struct AP64 { // Allocator64 parameters. Deliberately using a short name.109 static const uptr kSpaceBeg = 0x300000000000;110 static const uptr kSpaceSize = 0x020000000000; // 2T.111 static const uptr kMetadataSize = sizeof(Metadata);112 using SizeClassMap = DefaultSizeClassMap;113 using MapUnmapCallback = MsanMapUnmapCallback;114 static const uptr kFlags = 0;115 using AddressSpaceView = LocalAddressSpaceView;116};117 118using PrimaryAllocator = SizeClassAllocator64<AP64>;119#elif defined(__s390x__)120const uptr kMaxAllowedMallocSize = 2UL << 30; // 2G121 122struct AP64 { // Allocator64 parameters. Deliberately using a short name.123 static const uptr kSpaceBeg = 0x440000000000;124 static const uptr kSpaceSize = 0x020000000000; // 2T.125 static const uptr kMetadataSize = sizeof(Metadata);126 using SizeClassMap = DefaultSizeClassMap;127 using MapUnmapCallback = MsanMapUnmapCallback;128 static const uptr kFlags = 0;129 using AddressSpaceView = LocalAddressSpaceView;130};131 132using PrimaryAllocator = SizeClassAllocator64<AP64>;133#elif defined(__aarch64__)134const uptr kMaxAllowedMallocSize = 8UL << 30;135 136struct AP64 {137 static const uptr kSpaceBeg = 0xE00000000000ULL;138 static const uptr kSpaceSize = 0x40000000000; // 4T.139 static const uptr kMetadataSize = sizeof(Metadata);140 using SizeClassMap = DefaultSizeClassMap;141 using MapUnmapCallback = MsanMapUnmapCallback;142 static const uptr kFlags = 0;143 using AddressSpaceView = LocalAddressSpaceView;144};145using PrimaryAllocator = SizeClassAllocator64<AP64>;146#endif147using Allocator = CombinedAllocator<PrimaryAllocator>;148using AllocatorCache = Allocator::AllocatorCache;149} // namespace __msan150 151static Allocator allocator;152static AllocatorCache fallback_allocator_cache;153static StaticSpinMutex fallback_mutex;154 155static uptr max_malloc_size;156 157void __msan::MsanAllocatorInit() {158 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);159 allocator.Init(common_flags()->allocator_release_to_os_interval_ms);160 if (common_flags()->max_allocation_size_mb)161 max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,162 kMaxAllowedMallocSize);163 else164 max_malloc_size = kMaxAllowedMallocSize;165}166 167void __msan::LockAllocator() { allocator.ForceLock(); }168 169void __msan::UnlockAllocator() { allocator.ForceUnlock(); }170 171AllocatorCache *GetAllocatorCache(MsanThreadLocalMallocStorage *ms) {172 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));173 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);174}175 176void MsanThreadLocalMallocStorage::Init() {177 allocator.InitCache(GetAllocatorCache(this));178}179 180void MsanThreadLocalMallocStorage::CommitBack() {181 allocator.SwallowCache(GetAllocatorCache(this));182 allocator.DestroyCache(GetAllocatorCache(this));183}184 185static void *MsanAllocate(BufferedStackTrace *stack, uptr size, uptr alignment,186 bool zero) {187 if (UNLIKELY(size > max_malloc_size)) {188 if (AllocatorMayReturnNull()) {189 Report("WARNING: MemorySanitizer failed to allocate 0x%zx bytes\n", size);190 return nullptr;191 }192 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);193 ReportAllocationSizeTooBig(size, max_malloc_size, stack);194 }195 if (UNLIKELY(IsRssLimitExceeded())) {196 if (AllocatorMayReturnNull())197 return nullptr;198 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);199 ReportRssLimitExceeded(stack);200 }201 MsanThread *t = GetCurrentThread();202 void *allocated;203 if (t) {204 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());205 allocated = allocator.Allocate(cache, size, alignment);206 } else {207 SpinMutexLock l(&fallback_mutex);208 AllocatorCache *cache = &fallback_allocator_cache;209 allocated = allocator.Allocate(cache, size, alignment);210 }211 if (UNLIKELY(!allocated)) {212 SetAllocatorOutOfMemory();213 if (AllocatorMayReturnNull())214 return nullptr;215 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);216 ReportOutOfMemory(size, stack);217 }218 auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));219 meta->requested_size = size;220 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(allocated);221 void* padding_start = reinterpret_cast<char*>(allocated) + size;222 uptr padding_size = actually_allocated_size - size;223 224 // - With calloc(7,1), we can set the ideal tagging:225 // bytes 0-6: initialized, origin not set (and irrelevant)226 // byte 7: uninitialized, origin TAG_ALLOC_PADDING227 // bytes 8-15: uninitialized, origin TAG_ALLOC_PADDING228 // - If we have malloc(7) and __msan_get_track_origins() > 1, the 4-byte229 // origin granularity only allows the slightly suboptimal tagging:230 // bytes 0-6: uninitialized, origin TAG_ALLOC231 // byte 7: uninitialized, origin TAG_ALLOC (suboptimal)232 // bytes 8-15: uninitialized, origin TAG_ALLOC_PADDING233 // - If we have malloc(7) and __msan_get_track_origins() == 1, we use a234 // single origin bean to reduce overhead:235 // bytes 0-6: uninitialized, origin TAG_ALLOC236 // byte 7: uninitialized, origin TAG_ALLOC (suboptimal)237 // bytes 8-15: uninitialized, origin TAG_ALLOC (suboptimal)238 if (__msan_get_track_origins() && flags()->poison_in_malloc &&239 (zero || (__msan_get_track_origins() > 1))) {240 stack->tag = STACK_TRACE_TAG_ALLOC_PADDING;241 Origin o2 = Origin::CreateHeapOrigin(stack);242 __msan_set_origin(padding_start, padding_size, o2.raw_id());243 }244 245 if (zero) {246 if (allocator.FromPrimary(allocated))247 __msan_clear_and_unpoison(allocated, size);248 else249 __msan_unpoison(allocated, size); // Mem is already zeroed.250 251 if (flags()->poison_in_malloc)252 __msan_poison(padding_start, padding_size);253 } else if (flags()->poison_in_malloc) {254 __msan_poison(allocated, actually_allocated_size);255 256 if (__msan_get_track_origins()) {257 stack->tag = StackTrace::TAG_ALLOC;258 Origin o = Origin::CreateHeapOrigin(stack);259 __msan_set_origin(260 allocated,261 __msan_get_track_origins() == 1 ? actually_allocated_size : size,262 o.raw_id());263 }264 }265 266 UnpoisonParam(2);267 RunMallocHooks(allocated, size);268 return allocated;269}270 271void __msan::MsanDeallocate(BufferedStackTrace *stack, void *p) {272 DCHECK(p);273 UnpoisonParam(1);274 RunFreeHooks(p);275 276 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));277 uptr size = meta->requested_size;278 meta->requested_size = 0;279 // This memory will not be reused by anyone else, so we are free to keep it280 // poisoned. The secondary allocator will unmap and unpoison by281 // MsanMapUnmapCallback, no need to poison it here.282 if (flags()->poison_in_free && allocator.FromPrimary(p)) {283 __msan_poison(p, size);284 if (__msan_get_track_origins()) {285 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(p);286 stack->tag = StackTrace::TAG_DEALLOC;287 Origin o = Origin::CreateHeapOrigin(stack);288 __msan_set_origin(p, actually_allocated_size, o.raw_id());289 }290 }291 if (MsanThread *t = GetCurrentThread()) {292 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());293 allocator.Deallocate(cache, p);294 } else {295 SpinMutexLock l(&fallback_mutex);296 AllocatorCache *cache = &fallback_allocator_cache;297 allocator.Deallocate(cache, p);298 }299}300 301static void *MsanReallocate(BufferedStackTrace *stack, void *old_p,302 uptr new_size, uptr alignment) {303 Metadata *meta = reinterpret_cast<Metadata*>(allocator.GetMetaData(old_p));304 uptr old_size = meta->requested_size;305 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);306 if (new_size <= actually_allocated_size) {307 // We are not reallocating here.308 meta->requested_size = new_size;309 if (new_size > old_size) {310 if (flags()->poison_in_malloc) {311 stack->tag = StackTrace::TAG_ALLOC;312 PoisonMemory((char *)old_p + old_size, new_size - old_size, stack);313 }314 }315 return old_p;316 }317 uptr memcpy_size = Min(new_size, old_size);318 void *new_p = MsanAllocate(stack, new_size, alignment, false);319 if (new_p) {320 CopyMemory(new_p, old_p, memcpy_size, stack);321 MsanDeallocate(stack, old_p);322 }323 return new_p;324}325 326static void *MsanCalloc(BufferedStackTrace *stack, uptr nmemb, uptr size) {327 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {328 if (AllocatorMayReturnNull())329 return nullptr;330 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);331 ReportCallocOverflow(nmemb, size, stack);332 }333 return MsanAllocate(stack, nmemb * size, sizeof(u64), true);334}335 336static const void *AllocationBegin(const void *p) {337 if (!p)338 return nullptr;339 void *beg = allocator.GetBlockBegin(p);340 if (!beg)341 return nullptr;342 auto *b = reinterpret_cast<Metadata *>(allocator.GetMetaData(beg));343 if (!b)344 return nullptr;345 if (b->requested_size == 0)346 return nullptr;347 348 return beg;349}350 351static uptr AllocationSizeFast(const void *p) {352 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;353}354 355static uptr AllocationSize(const void *p) {356 if (!p)357 return 0;358 if (allocator.GetBlockBegin(p) != p)359 return 0;360 return AllocationSizeFast(p);361}362 363void *__msan::msan_malloc(uptr size, BufferedStackTrace *stack) {364 return SetErrnoOnNull(MsanAllocate(stack, size, sizeof(u64), false));365}366 367void *__msan::msan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {368 return SetErrnoOnNull(MsanCalloc(stack, nmemb, size));369}370 371void *__msan::msan_realloc(void *ptr, uptr size, BufferedStackTrace *stack) {372 if (!ptr)373 return SetErrnoOnNull(MsanAllocate(stack, size, sizeof(u64), false));374 if (size == 0) {375 MsanDeallocate(stack, ptr);376 return nullptr;377 }378 return SetErrnoOnNull(MsanReallocate(stack, ptr, size, sizeof(u64)));379}380 381void *__msan::msan_reallocarray(void *ptr, uptr nmemb, uptr size,382 BufferedStackTrace *stack) {383 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {384 errno = errno_ENOMEM;385 if (AllocatorMayReturnNull())386 return nullptr;387 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);388 ReportReallocArrayOverflow(nmemb, size, stack);389 }390 return msan_realloc(ptr, nmemb * size, stack);391}392 393void *__msan::msan_valloc(uptr size, BufferedStackTrace *stack) {394 return SetErrnoOnNull(MsanAllocate(stack, size, GetPageSizeCached(), false));395}396 397void *__msan::msan_pvalloc(uptr size, BufferedStackTrace *stack) {398 uptr PageSize = GetPageSizeCached();399 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {400 errno = errno_ENOMEM;401 if (AllocatorMayReturnNull())402 return nullptr;403 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);404 ReportPvallocOverflow(size, stack);405 }406 // pvalloc(0) should allocate one page.407 size = size ? RoundUpTo(size, PageSize) : PageSize;408 return SetErrnoOnNull(MsanAllocate(stack, size, PageSize, false));409}410 411void *__msan::msan_aligned_alloc(uptr alignment, uptr size,412 BufferedStackTrace *stack) {413 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {414 errno = errno_EINVAL;415 if (AllocatorMayReturnNull())416 return nullptr;417 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);418 ReportInvalidAlignedAllocAlignment(size, alignment, stack);419 }420 return SetErrnoOnNull(MsanAllocate(stack, size, alignment, false));421}422 423void *__msan::msan_memalign(uptr alignment, uptr size,424 BufferedStackTrace *stack) {425 if (UNLIKELY(!IsPowerOfTwo(alignment))) {426 errno = errno_EINVAL;427 if (AllocatorMayReturnNull())428 return nullptr;429 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);430 ReportInvalidAllocationAlignment(alignment, stack);431 }432 return SetErrnoOnNull(MsanAllocate(stack, size, alignment, false));433}434 435int __msan::msan_posix_memalign(void **memptr, uptr alignment, uptr size,436 BufferedStackTrace *stack) {437 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {438 if (AllocatorMayReturnNull())439 return errno_EINVAL;440 GET_FATAL_STACK_TRACE_IF_EMPTY(stack);441 ReportInvalidPosixMemalignAlignment(alignment, stack);442 }443 void *ptr = MsanAllocate(stack, size, alignment, false);444 if (UNLIKELY(!ptr))445 // OOM error is already taken care of by MsanAllocate.446 return errno_ENOMEM;447 CHECK(IsAligned((uptr)ptr, alignment));448 *memptr = ptr;449 return 0;450}451 452extern "C" {453uptr __sanitizer_get_current_allocated_bytes() {454 uptr stats[AllocatorStatCount];455 allocator.GetStats(stats);456 return stats[AllocatorStatAllocated];457}458 459uptr __sanitizer_get_heap_size() {460 uptr stats[AllocatorStatCount];461 allocator.GetStats(stats);462 return stats[AllocatorStatMapped];463}464 465uptr __sanitizer_get_free_bytes() { return 1; }466 467uptr __sanitizer_get_unmapped_bytes() { return 1; }468 469uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }470 471int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }472 473const void *__sanitizer_get_allocated_begin(const void *p) {474 return AllocationBegin(p);475}476 477uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }478 479uptr __sanitizer_get_allocated_size_fast(const void *p) {480 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));481 uptr ret = AllocationSizeFast(p);482 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));483 return ret;484}485 486void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }487}488