215 lines · cpp
1//===-- sanitizer_allocator.cpp -------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is shared between AddressSanitizer and ThreadSanitizer10// run-time libraries.11// This allocator is used inside run-times.12//===----------------------------------------------------------------------===//13 14#include "sanitizer_allocator.h"15 16#include "sanitizer_allocator_checks.h"17#include "sanitizer_allocator_internal.h"18#include "sanitizer_atomic.h"19#include "sanitizer_common.h"20#include "sanitizer_platform.h"21 22namespace __sanitizer {23 24// Default allocator names.25const char *PrimaryAllocatorName = "SizeClassAllocator";26const char *SecondaryAllocatorName = "LargeMmapAllocator";27 28alignas(64) static char internal_alloc_placeholder[sizeof(InternalAllocator)];29static atomic_uint8_t internal_allocator_initialized;30static StaticSpinMutex internal_alloc_init_mu;31 32static InternalAllocatorCache internal_allocator_cache;33static StaticSpinMutex internal_allocator_cache_mu;34 35InternalAllocator *internal_allocator() {36 InternalAllocator *internal_allocator_instance =37 reinterpret_cast<InternalAllocator *>(&internal_alloc_placeholder);38 if (atomic_load(&internal_allocator_initialized, memory_order_acquire) == 0) {39 SpinMutexLock l(&internal_alloc_init_mu);40 if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) ==41 0) {42 internal_allocator_instance->Init(kReleaseToOSIntervalNever);43 atomic_store(&internal_allocator_initialized, 1, memory_order_release);44 }45 }46 return internal_allocator_instance;47}48 49static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache,50 uptr alignment) {51 if (alignment == 0) alignment = 8;52 if (cache == 0) {53 SpinMutexLock l(&internal_allocator_cache_mu);54 return internal_allocator()->Allocate(&internal_allocator_cache, size,55 alignment);56 }57 return internal_allocator()->Allocate(cache, size, alignment);58}59 60static void *RawInternalRealloc(void *ptr, uptr size,61 InternalAllocatorCache *cache) {62 constexpr usize alignment = Max<usize>(8, sizeof(void *));63 if (cache == 0) {64 SpinMutexLock l(&internal_allocator_cache_mu);65 return internal_allocator()->Reallocate(&internal_allocator_cache, ptr,66 size, alignment);67 }68 return internal_allocator()->Reallocate(cache, ptr, size, alignment);69}70 71static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) {72 if (!cache) {73 SpinMutexLock l(&internal_allocator_cache_mu);74 return internal_allocator()->Deallocate(&internal_allocator_cache, ptr);75 }76 internal_allocator()->Deallocate(cache, ptr);77}78 79static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) {80 SetAllocatorOutOfMemory();81 Report("FATAL: %s: internal allocator is out of memory trying to allocate "82 "0x%zx bytes\n", SanitizerToolName, requested_size);83 Die();84}85 86void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) {87 void *p = RawInternalAlloc(size, cache, alignment);88 if (UNLIKELY(!p))89 ReportInternalAllocatorOutOfMemory(size);90 return p;91}92 93void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) {94 void *p = RawInternalRealloc(addr, size, cache);95 if (UNLIKELY(!p))96 ReportInternalAllocatorOutOfMemory(size);97 return p;98}99 100void *InternalReallocArray(void *addr, uptr count, uptr size,101 InternalAllocatorCache *cache) {102 if (UNLIKELY(CheckForCallocOverflow(count, size))) {103 Report(104 "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) "105 "cannot be represented in type size_t\n",106 SanitizerToolName, count, size);107 Die();108 }109 return InternalRealloc(addr, count * size, cache);110}111 112void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) {113 if (UNLIKELY(CheckForCallocOverflow(count, size))) {114 Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) "115 "cannot be represented in type size_t\n", SanitizerToolName, count,116 size);117 Die();118 }119 void *p = InternalAlloc(count * size, cache);120 if (LIKELY(p))121 internal_memset(p, 0, count * size);122 return p;123}124 125void InternalFree(void *addr, InternalAllocatorCache *cache) {126 RawInternalFree(addr, cache);127}128 129void InternalAllocatorLock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {130 internal_allocator_cache_mu.Lock();131 internal_allocator()->ForceLock();132}133 134void InternalAllocatorUnlock() SANITIZER_NO_THREAD_SAFETY_ANALYSIS {135 internal_allocator()->ForceUnlock();136 internal_allocator_cache_mu.Unlock();137}138 139// LowLevelAllocator140constexpr usize kLowLevelAllocatorDefaultAlignment =141 Max<usize>(8, sizeof(void *));142constexpr uptr kMinNumPagesRounded = 16;143constexpr uptr kMinRoundedSize = 65536;144static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment;145static LowLevelAllocateCallback low_level_alloc_callback;146 147static LowLevelAllocator Alloc;148LowLevelAllocator &GetGlobalLowLevelAllocator() { return Alloc; }149 150void *LowLevelAllocator::Allocate(uptr size) {151 // Align allocation size.152 size = RoundUpTo(size, low_level_alloc_min_alignment);153 if (allocated_end_ - allocated_current_ < (sptr)size) {154 uptr size_to_allocate = RoundUpTo(155 size, Min(GetPageSizeCached() * kMinNumPagesRounded, kMinRoundedSize));156 allocated_current_ = (char *)MmapOrDie(size_to_allocate, __func__);157 allocated_end_ = allocated_current_ + size_to_allocate;158 if (low_level_alloc_callback) {159 low_level_alloc_callback((uptr)allocated_current_, size_to_allocate);160 }161 }162 CHECK(allocated_end_ - allocated_current_ >= (sptr)size);163 void *res = allocated_current_;164 allocated_current_ += size;165 return res;166}167 168void SetLowLevelAllocateMinAlignment(uptr alignment) {169 CHECK(IsPowerOfTwo(alignment));170 low_level_alloc_min_alignment = Max(alignment, low_level_alloc_min_alignment);171}172 173void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) {174 low_level_alloc_callback = callback;175}176 177// Allocator's OOM and other errors handling support.178 179static atomic_uint8_t allocator_out_of_memory = {0};180static atomic_uint8_t allocator_may_return_null = {0};181 182bool IsAllocatorOutOfMemory() {183 return atomic_load_relaxed(&allocator_out_of_memory);184}185 186void SetAllocatorOutOfMemory() {187 atomic_store_relaxed(&allocator_out_of_memory, 1);188}189 190bool AllocatorMayReturnNull() {191 return atomic_load(&allocator_may_return_null, memory_order_relaxed);192}193 194void SetAllocatorMayReturnNull(bool may_return_null) {195 atomic_store(&allocator_may_return_null, may_return_null,196 memory_order_relaxed);197}198 199void PrintHintAllocatorCannotReturnNull() {200 Report("HINT: if you don't care about these errors you may set "201 "allocator_may_return_null=1\n");202}203 204static atomic_uint8_t rss_limit_exceeded;205 206bool IsRssLimitExceeded() {207 return atomic_load(&rss_limit_exceeded, memory_order_relaxed);208}209 210void SetRssLimitExceeded(bool limit_exceeded) {211 atomic_store(&rss_limit_exceeded, limit_exceeded, memory_order_relaxed);212}213 214} // namespace __sanitizer215