335 lines · cpp
1//===-- dfsan_allocator.cpp -------------------------- --------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a part of DataflowSanitizer.10//11// DataflowSanitizer allocator.12//===----------------------------------------------------------------------===//13 14#include "dfsan_allocator.h"15 16#include "dfsan.h"17#include "dfsan_flags.h"18#include "dfsan_thread.h"19#include "sanitizer_common/sanitizer_allocator.h"20#include "sanitizer_common/sanitizer_allocator_checks.h"21#include "sanitizer_common/sanitizer_allocator_interface.h"22#include "sanitizer_common/sanitizer_allocator_report.h"23#include "sanitizer_common/sanitizer_errno.h"24 25using namespace __dfsan;26 27namespace {28 29struct Metadata {30 uptr requested_size;31};32 33struct DFsanMapUnmapCallback {34 void OnMap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }35 void OnMapSecondary(uptr p, uptr size, uptr user_begin,36 uptr user_size) const {37 OnMap(p, size);38 }39 void OnUnmap(uptr p, uptr size) const { dfsan_set_label(0, (void *)p, size); }40};41 42// Note: to ensure that the allocator is compatible with the application memory43// layout (especially with high-entropy ASLR), kSpaceBeg and kSpaceSize must be44// duplicated as MappingDesc::ALLOCATOR in dfsan_platform.h.45#if defined(__aarch64__)46const uptr kAllocatorSpace = 0xE00000000000ULL;47#else48const uptr kAllocatorSpace = 0x700000000000ULL;49#endif50const uptr kMaxAllowedMallocSize = 1ULL << 40;51 52struct AP64 { // Allocator64 parameters. Deliberately using a short name.53 static const uptr kSpaceBeg = kAllocatorSpace;54 static const uptr kSpaceSize = 0x40000000000; // 4T.55 static const uptr kMetadataSize = sizeof(Metadata);56 typedef DefaultSizeClassMap SizeClassMap;57 typedef DFsanMapUnmapCallback MapUnmapCallback;58 static const uptr kFlags = 0;59 using AddressSpaceView = LocalAddressSpaceView;60};61 62typedef SizeClassAllocator64<AP64> PrimaryAllocator;63 64typedef CombinedAllocator<PrimaryAllocator> Allocator;65typedef Allocator::AllocatorCache AllocatorCache;66 67static Allocator allocator;68static AllocatorCache fallback_allocator_cache;69static StaticSpinMutex fallback_mutex;70 71static uptr max_malloc_size;72} // namespace73 74void __dfsan::dfsan_allocator_init() {75 SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);76 allocator.Init(common_flags()->allocator_release_to_os_interval_ms);77 if (common_flags()->max_allocation_size_mb)78 max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,79 kMaxAllowedMallocSize);80 else81 max_malloc_size = kMaxAllowedMallocSize;82}83 84static AllocatorCache *GetAllocatorCache(DFsanThreadLocalMallocStorage *ms) {85 CHECK(ms);86 CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));87 return reinterpret_cast<AllocatorCache *>(ms->allocator_cache);88}89 90void DFsanThreadLocalMallocStorage::CommitBack() {91 allocator.SwallowCache(GetAllocatorCache(this));92}93 94static void *DFsanAllocate(uptr size, uptr alignment, bool zeroise) {95 if (size > max_malloc_size) {96 if (AllocatorMayReturnNull()) {97 Report("WARNING: DataflowSanitizer failed to allocate 0x%zx bytes\n",98 size);99 return nullptr;100 }101 UNINITIALIZED BufferedStackTrace stack;102 ReportAllocationSizeTooBig(size, max_malloc_size, &stack);103 }104 if (UNLIKELY(IsRssLimitExceeded())) {105 if (AllocatorMayReturnNull())106 return nullptr;107 UNINITIALIZED BufferedStackTrace stack;108 ReportRssLimitExceeded(&stack);109 }110 DFsanThread *t = GetCurrentThread();111 void *allocated;112 if (t) {113 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());114 allocated = allocator.Allocate(cache, size, alignment);115 } else {116 SpinMutexLock l(&fallback_mutex);117 AllocatorCache *cache = &fallback_allocator_cache;118 allocated = allocator.Allocate(cache, size, alignment);119 }120 if (UNLIKELY(!allocated)) {121 SetAllocatorOutOfMemory();122 if (AllocatorMayReturnNull())123 return nullptr;124 UNINITIALIZED BufferedStackTrace stack;125 ReportOutOfMemory(size, &stack);126 }127 Metadata *meta =128 reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));129 meta->requested_size = size;130 if (zeroise) {131 internal_memset(allocated, 0, size);132 dfsan_set_label(0, allocated, size);133 } else if (flags().zero_in_malloc) {134 dfsan_set_label(0, allocated, size);135 }136 return allocated;137}138 139void __dfsan::dfsan_deallocate(void *p) {140 CHECK(p);141 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(p));142 uptr size = meta->requested_size;143 meta->requested_size = 0;144 if (flags().zero_in_free)145 dfsan_set_label(0, p, size);146 DFsanThread *t = GetCurrentThread();147 if (t) {148 AllocatorCache *cache = GetAllocatorCache(&t->malloc_storage());149 allocator.Deallocate(cache, p);150 } else {151 SpinMutexLock l(&fallback_mutex);152 AllocatorCache *cache = &fallback_allocator_cache;153 allocator.Deallocate(cache, p);154 }155}156 157static void *DFsanReallocate(void *old_p, uptr new_size, uptr alignment) {158 Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(old_p));159 uptr old_size = meta->requested_size;160 uptr actually_allocated_size = allocator.GetActuallyAllocatedSize(old_p);161 if (new_size <= actually_allocated_size) {162 // We are not reallocating here.163 meta->requested_size = new_size;164 if (new_size > old_size && flags().zero_in_malloc)165 dfsan_set_label(0, (char *)old_p + old_size, new_size - old_size);166 return old_p;167 }168 uptr memcpy_size = Min(new_size, old_size);169 void *new_p = DFsanAllocate(new_size, alignment, false /*zeroise*/);170 if (new_p) {171 dfsan_copy_memory(new_p, old_p, memcpy_size);172 dfsan_deallocate(old_p);173 }174 return new_p;175}176 177static void *DFsanCalloc(uptr nmemb, uptr size) {178 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {179 if (AllocatorMayReturnNull())180 return nullptr;181 UNINITIALIZED BufferedStackTrace stack;182 ReportCallocOverflow(nmemb, size, &stack);183 }184 return DFsanAllocate(nmemb * size, sizeof(u64), true /*zeroise*/);185}186 187static const void *AllocationBegin(const void *p) {188 if (!p)189 return nullptr;190 void *beg = allocator.GetBlockBegin(p);191 if (!beg)192 return nullptr;193 Metadata *b = (Metadata *)allocator.GetMetaData(beg);194 if (!b)195 return nullptr;196 if (b->requested_size == 0)197 return nullptr;198 return (const void *)beg;199}200 201static uptr AllocationSize(const void *p) {202 if (!p)203 return 0;204 const void *beg = allocator.GetBlockBegin(p);205 if (beg != p)206 return 0;207 Metadata *b = (Metadata *)allocator.GetMetaData(p);208 return b->requested_size;209}210 211static uptr AllocationSizeFast(const void *p) {212 return reinterpret_cast<Metadata *>(allocator.GetMetaData(p))->requested_size;213}214 215void *__dfsan::dfsan_malloc(uptr size) {216 return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));217}218 219void *__dfsan::dfsan_calloc(uptr nmemb, uptr size) {220 return SetErrnoOnNull(DFsanCalloc(nmemb, size));221}222 223void *__dfsan::dfsan_realloc(void *ptr, uptr size) {224 if (!ptr)225 return SetErrnoOnNull(DFsanAllocate(size, sizeof(u64), false /*zeroise*/));226 if (size == 0) {227 dfsan_deallocate(ptr);228 return nullptr;229 }230 return SetErrnoOnNull(DFsanReallocate(ptr, size, sizeof(u64)));231}232 233void *__dfsan::dfsan_reallocarray(void *ptr, uptr nmemb, uptr size) {234 if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {235 errno = errno_ENOMEM;236 if (AllocatorMayReturnNull())237 return nullptr;238 UNINITIALIZED BufferedStackTrace stack;239 ReportReallocArrayOverflow(nmemb, size, &stack);240 }241 return dfsan_realloc(ptr, nmemb * size);242}243 244void *__dfsan::dfsan_valloc(uptr size) {245 return SetErrnoOnNull(246 DFsanAllocate(size, GetPageSizeCached(), false /*zeroise*/));247}248 249void *__dfsan::dfsan_pvalloc(uptr size) {250 uptr PageSize = GetPageSizeCached();251 if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {252 errno = errno_ENOMEM;253 if (AllocatorMayReturnNull())254 return nullptr;255 UNINITIALIZED BufferedStackTrace stack;256 ReportPvallocOverflow(size, &stack);257 }258 // pvalloc(0) should allocate one page.259 size = size ? RoundUpTo(size, PageSize) : PageSize;260 return SetErrnoOnNull(DFsanAllocate(size, PageSize, false /*zeroise*/));261}262 263void *__dfsan::dfsan_aligned_alloc(uptr alignment, uptr size) {264 if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {265 errno = errno_EINVAL;266 if (AllocatorMayReturnNull())267 return nullptr;268 UNINITIALIZED BufferedStackTrace stack;269 ReportInvalidAlignedAllocAlignment(size, alignment, &stack);270 }271 return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));272}273 274void *__dfsan::dfsan_memalign(uptr alignment, uptr size) {275 if (UNLIKELY(!IsPowerOfTwo(alignment))) {276 errno = errno_EINVAL;277 if (AllocatorMayReturnNull())278 return nullptr;279 UNINITIALIZED BufferedStackTrace stack;280 ReportInvalidAllocationAlignment(alignment, &stack);281 }282 return SetErrnoOnNull(DFsanAllocate(size, alignment, false /*zeroise*/));283}284 285int __dfsan::dfsan_posix_memalign(void **memptr, uptr alignment, uptr size) {286 if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {287 if (AllocatorMayReturnNull())288 return errno_EINVAL;289 UNINITIALIZED BufferedStackTrace stack;290 ReportInvalidPosixMemalignAlignment(alignment, &stack);291 }292 void *ptr = DFsanAllocate(size, alignment, false /*zeroise*/);293 if (UNLIKELY(!ptr))294 // OOM error is already taken care of by DFsanAllocate.295 return errno_ENOMEM;296 CHECK(IsAligned((uptr)ptr, alignment));297 *memptr = ptr;298 return 0;299}300 301extern "C" {302uptr __sanitizer_get_current_allocated_bytes() {303 uptr stats[AllocatorStatCount];304 allocator.GetStats(stats);305 return stats[AllocatorStatAllocated];306}307 308uptr __sanitizer_get_heap_size() {309 uptr stats[AllocatorStatCount];310 allocator.GetStats(stats);311 return stats[AllocatorStatMapped];312}313 314uptr __sanitizer_get_free_bytes() { return 1; }315 316uptr __sanitizer_get_unmapped_bytes() { return 1; }317 318uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }319 320int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }321 322const void *__sanitizer_get_allocated_begin(const void *p) {323 return AllocationBegin(p);324}325 326uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }327 328uptr __sanitizer_get_allocated_size_fast(const void *p) {329 DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));330 uptr ret = AllocationSizeFast(p);331 DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));332 return ret;333}334}335