341 lines · cpp
1//===- nsan_allocator.cpp -------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// NumericalStabilitySanitizer allocator.10//11//===----------------------------------------------------------------------===//12 13#include "nsan_allocator.h"14#include "interception/interception.h"15#include "nsan.h"16#include "nsan_flags.h"17#include "nsan_platform.h"18#include "nsan_thread.h"19#include "sanitizer_common/sanitizer_allocator.h"20#include "sanitizer_common/sanitizer_allocator_checks.h"21#include "sanitizer_common/sanitizer_allocator_interface.h"22#include "sanitizer_common/sanitizer_allocator_report.h"23#include "sanitizer_common/sanitizer_common.h"24#include "sanitizer_common/sanitizer_errno.h"25 26using namespace __nsan;27 28DECLARE_REAL(void *, memcpy, void *dest, const void *src, SIZE_T n)29DECLARE_REAL(void *, memset, void *dest, int c, SIZE_T n)30 31namespace {32struct Metadata {33 uptr requested_size;34};35 36struct NsanMapUnmapCallback {37 void OnMap(uptr p, uptr size) const {}38 void OnMapSecondary(uptr p, uptr size, uptr user_begin,39 uptr user_size) const {}40 void OnUnmap(uptr p, uptr size) const {}41};42 43const uptr kMaxAllowedMallocSize = 1ULL << 40;44 45// Allocator64 parameters. Deliberately using a short name.46struct AP64 {47 static const uptr kSpaceBeg = Mapping::kHeapMemBeg;48 static const uptr kSpaceSize = 0x40000000000; // 4T.49 static const uptr kMetadataSize = sizeof(Metadata);50 using SizeClassMap = DefaultSizeClassMap;51 using MapUnmapCallback = NsanMapUnmapCallback;52 static const uptr kFlags = 0;53 using AddressSpaceView = LocalAddressSpaceView;54};55} // namespace56 57using PrimaryAllocator = SizeClassAllocator64<AP64>;58using Allocator = CombinedAllocator<PrimaryAllocator>;59using AllocatorCache = Allocator::AllocatorCache;60 61static Allocator allocator;62static AllocatorCache fallback_allocator_cache;63static StaticSpinMutex fallback_mutex;64 65static uptr max_malloc_size;66 67void __nsan::NsanAllocatorInit() {68 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);69 allocator.Init(common_flags()->allocator_release_to_os_interval_ms);70 if (common_flags()->max_allocation_size_mb)71 max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,72 kMaxAllowedMallocSize);73 else74 max_malloc_size = kMaxAllowedMallocSize;75}76 77static AllocatorCache *GetAllocatorCache(NsanThreadLocalMallocStorage *ms) {78 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));79 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);80}81 82void NsanThreadLocalMallocStorage::Init() {83 allocator.InitCache(GetAllocatorCache(this));84}85 86void NsanThreadLocalMallocStorage::CommitBack() {87 allocator.SwallowCache(GetAllocatorCache(this));88 allocator.DestroyCache(GetAllocatorCache(this));89}90 91static void *NsanAllocate(uptr size, uptr alignment, bool zero) {92 if (UNLIKELY(size > max_malloc_size)) {93 if (AllocatorMayReturnNull()) {94 Report("WARNING: NumericalStabilitySanitizer failed to allocate 0x%zx "95 "bytes\n",96 size);97 return nullptr;98 }99 BufferedStackTrace stack;100 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);101 ReportAllocationSizeTooBig(size, max_malloc_size, &stack);102 }103 if (UNLIKELY(IsRssLimitExceeded())) {104 if (AllocatorMayReturnNull())105 return nullptr;106 BufferedStackTrace stack;107 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);108 ReportRssLimitExceeded(&stack);109 }110 111 void *allocated;112 if (NsanThread *t = GetCurrentThread()) {113 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());114 allocated = allocator.Allocate(cache, size, alignment);115 } else {116 SpinMutexLock l(&fallback_mutex);117 AllocatorCache *cache = &fallback_allocator_cache;118 allocated = allocator.Allocate(cache, size, alignment);119 }120 if (UNLIKELY(!allocated)) {121 SetAllocatorOutOfMemory();122 if (AllocatorMayReturnNull())123 return nullptr;124 BufferedStackTrace stack;125 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);126 ReportOutOfMemory(size, &stack);127 }128 auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));129 meta->requested_size = size;130 if (zero && allocator.FromPrimary(allocated))131 REAL(memset)(allocated, 0, size);132 __nsan_set_value_unknown(allocated, size);133 RunMallocHooks(allocated, size);134 return allocated;135}136 137void __nsan::NsanDeallocate(void *p) {138 DCHECK(p);139 RunFreeHooks(p);140 auto *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));141 uptr size = meta->requested_size;142 meta->requested_size = 0;143 if (flags().poison_in_free)144 __nsan_set_value_unknown(p, size);145 if (NsanThread *t = GetCurrentThread()) {146 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());147 allocator.Deallocate(cache, p);148 } else {149 // In a just created thread, glibc's _dl_deallocate_tls might reach here150 // before nsan_current_thread is set.151 SpinMutexLock l(&fallback_mutex);152 AllocatorCache *cache = &fallback_allocator_cache;153 allocator.Deallocate(cache, p);154 }155}156 157static void *NsanReallocate(void *ptr, uptr new_size, uptr alignment) {158 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(ptr));159 uptr old_size = meta->requested_size;160 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(ptr);161 if (new_size <= actually_allocated_size) {162 // We are not reallocating here.163 meta->requested_size = new_size;164 if (new_size > old_size)165 __nsan_set_value_unknown((u8 *)ptr + old_size, new_size - old_size);166 return ptr;167 }168 void *new_p = NsanAllocate(new_size, alignment, false);169 if (new_p) {170 uptr memcpy_size = Min(new_size, old_size);171 REAL(memcpy)(new_p, ptr, memcpy_size);172 __nsan_copy_values(new_p, ptr, memcpy_size);173 NsanDeallocate(ptr);174 }175 return new_p;176}177 178static void *NsanCalloc(uptr nmemb, uptr size) {179 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {180 if (AllocatorMayReturnNull())181 return nullptr;182 BufferedStackTrace stack;183 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);184 ReportCallocOverflow(nmemb, size, &stack);185 }186 return NsanAllocate(nmemb * size, sizeof(u64), true);187}188 189static const void *AllocationBegin(const void *p) {190 if (!p)191 return nullptr;192 void *beg = allocator.GetBlockBegin(p);193 if (!beg)194 return nullptr;195 auto *b = reinterpret_cast<Metadata *>(allocator.GetMetaData(beg));196 if (!b)197 return nullptr;198 if (b->requested_size == 0)199 return nullptr;200 201 return beg;202}203 204static uptr AllocationSizeFast(const void *p) {205 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;206}207 208static uptr AllocationSize(const void *p) {209 if (!p)210 return 0;211 if (allocator.GetBlockBegin(p) != p)212 return 0;213 return AllocationSizeFast(p);214}215 216void *__nsan::nsan_malloc(uptr size) {217 return SetErrnoOnNull(NsanAllocate(size, sizeof(u64), false));218}219 220void *__nsan::nsan_calloc(uptr nmemb, uptr size) {221 return SetErrnoOnNull(NsanCalloc(nmemb, size));222}223 224void *__nsan::nsan_realloc(void *ptr, uptr size) {225 if (!ptr)226 return SetErrnoOnNull(NsanAllocate(size, sizeof(u64), false));227 if (size == 0) {228 NsanDeallocate(ptr);229 return nullptr;230 }231 return SetErrnoOnNull(NsanReallocate(ptr, size, sizeof(u64)));232}233 234void *__nsan::nsan_reallocarray(void *ptr, uptr nmemb, uptr size) {235 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {236 errno = errno_ENOMEM;237 if (AllocatorMayReturnNull())238 return nullptr;239 BufferedStackTrace stack;240 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);241 ReportReallocArrayOverflow(nmemb, size, &stack);242 }243 return nsan_realloc(ptr, nmemb * size);244}245 246void *__nsan::nsan_valloc(uptr size) {247 return SetErrnoOnNull(NsanAllocate(size, GetPageSizeCached(), false));248}249 250void *__nsan::nsan_pvalloc(uptr size) {251 uptr PageSize = GetPageSizeCached();252 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {253 errno = errno_ENOMEM;254 if (AllocatorMayReturnNull())255 return nullptr;256 BufferedStackTrace stack;257 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);258 ReportPvallocOverflow(size, &stack);259 }260 // pvalloc(0) should allocate one page.261 size = size ? RoundUpTo(size, PageSize) : PageSize;262 return SetErrnoOnNull(NsanAllocate(size, PageSize, false));263}264 265void *__nsan::nsan_aligned_alloc(uptr alignment, uptr size) {266 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {267 errno = errno_EINVAL;268 if (AllocatorMayReturnNull())269 return nullptr;270 BufferedStackTrace stack;271 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);272 ReportInvalidAlignedAllocAlignment(size, alignment, &stack);273 }274 return SetErrnoOnNull(NsanAllocate(size, alignment, false));275}276 277void *__nsan::nsan_memalign(uptr alignment, uptr size) {278 if (UNLIKELY(!IsPowerOfTwo(alignment))) {279 errno = errno_EINVAL;280 if (AllocatorMayReturnNull())281 return nullptr;282 BufferedStackTrace stack;283 GET_FATAL_STACK_TRACE_IF_EMPTY(&stack);284 ReportInvalidAllocationAlignment(alignment, &stack);285 }286 return SetErrnoOnNull(NsanAllocate(size, alignment, false));287}288 289int __nsan::nsan_posix_memalign(void **memptr, uptr alignment, uptr size) {290 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {291 if (AllocatorMayReturnNull())292 return errno_EINVAL;293 BufferedStackTrace stack;294 ReportInvalidPosixMemalignAlignment(alignment, &stack);295 }296 void *ptr = NsanAllocate(size, alignment, false);297 if (UNLIKELY(!ptr))298 // OOM error is already taken care of by NsanAllocate.299 return errno_ENOMEM;300 DCHECK(IsAligned((uptr)ptr, alignment));301 *memptr = ptr;302 return 0;303}304 305extern "C" {306uptr __sanitizer_get_current_allocated_bytes() {307 uptr stats[AllocatorStatCount];308 allocator.GetStats(stats);309 return stats[AllocatorStatAllocated];310}311 312uptr __sanitizer_get_heap_size() {313 uptr stats[AllocatorStatCount];314 allocator.GetStats(stats);315 return stats[AllocatorStatMapped];316}317 318uptr __sanitizer_get_free_bytes() { return 1; }319 320uptr __sanitizer_get_unmapped_bytes() { return 1; }321 322uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }323 324int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }325 326const void *__sanitizer_get_allocated_begin(const void *p) {327 return AllocationBegin(p);328}329 330uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }331 332uptr __sanitizer_get_allocated_size_fast(const void *p) {333 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));334 uptr ret = AllocationSizeFast(p);335 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));336 return ret;337}338 339void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }340}341