378 lines · plain
1//===-- wrappers_c.inc ------------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#ifndef SCUDO_PREFIX10#error "Define SCUDO_PREFIX prior to including this file!"11#endif12 13// malloc-type functions have to be aligned to std::max_align_t. This is14// distinct from (1U << SCUDO_MIN_ALIGNMENT_LOG), since C++ new-type functions15// do not have to abide by the same requirement.16#ifndef SCUDO_MALLOC_ALIGNMENT17#define SCUDO_MALLOC_ALIGNMENT FIRST_32_SECOND_64(8U, 16U)18#endif19 20static void reportAllocation(void *ptr, size_t size) {21 if (SCUDO_ENABLE_HOOKS)22 if (__scudo_allocate_hook && ptr)23 __scudo_allocate_hook(ptr, size);24}25static void reportDeallocation(void *ptr) {26 if (SCUDO_ENABLE_HOOKS)27 if (__scudo_deallocate_hook)28 __scudo_deallocate_hook(ptr);29}30static void reportReallocAllocation(void *old_ptr, void *new_ptr, size_t size) {31 DCHECK_NE(new_ptr, nullptr);32 33 if (SCUDO_ENABLE_HOOKS) {34 if (__scudo_realloc_allocate_hook)35 __scudo_realloc_allocate_hook(old_ptr, new_ptr, size);36 else if (__scudo_allocate_hook)37 __scudo_allocate_hook(new_ptr, size);38 }39}40static void reportReallocDeallocation(void *old_ptr) {41 if (SCUDO_ENABLE_HOOKS) {42 if (__scudo_realloc_deallocate_hook)43 __scudo_realloc_deallocate_hook(old_ptr);44 else if (__scudo_deallocate_hook)45 __scudo_deallocate_hook(old_ptr);46 }47}48 49extern "C" {50 51INTERFACE WEAK void *SCUDO_PREFIX(calloc)(size_t nmemb, size_t size) {52 scudo::uptr Product;53 if (UNLIKELY(scudo::checkForCallocOverflow(size, nmemb, &Product))) {54 if (SCUDO_ALLOCATOR.canReturnNull()) {55 errno = ENOMEM;56 return nullptr;57 }58 scudo::reportCallocOverflow(nmemb, size);59 }60 void *Ptr = SCUDO_ALLOCATOR.allocate(Product, scudo::Chunk::Origin::Malloc,61 SCUDO_MALLOC_ALIGNMENT, true);62 reportAllocation(Ptr, Product);63 return scudo::setErrnoOnNull(Ptr);64}65 66INTERFACE WEAK void SCUDO_PREFIX(free)(void *ptr) {67 reportDeallocation(ptr);68 SCUDO_ALLOCATOR.deallocate(ptr, scudo::Chunk::Origin::Malloc);69}70 71INTERFACE WEAK struct SCUDO_MALLINFO SCUDO_PREFIX(mallinfo)(void) {72 struct SCUDO_MALLINFO Info = {};73 scudo::StatCounters Stats;74 SCUDO_ALLOCATOR.getStats(Stats);75 // Space allocated in mmapped regions (bytes)76 Info.hblkhd = static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatMapped]);77 // Maximum total allocated space (bytes)78 Info.usmblks = Info.hblkhd;79 // Space in freed fastbin blocks (bytes)80 Info.fsmblks = static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatFree]);81 // Total allocated space (bytes)82 Info.uordblks =83 static_cast<__scudo_mallinfo_data_t>(Stats[scudo::StatAllocated]);84 // Total free space (bytes)85 Info.fordblks = Info.fsmblks;86 return Info;87}88 89// On Android, mallinfo2 is an alias of mallinfo, so don't define both.90#if !SCUDO_ANDROID91INTERFACE WEAK struct __scudo_mallinfo2 SCUDO_PREFIX(mallinfo2)(void) {92 struct __scudo_mallinfo2 Info = {};93 scudo::StatCounters Stats;94 SCUDO_ALLOCATOR.getStats(Stats);95 // Space allocated in mmapped regions (bytes)96 Info.hblkhd = Stats[scudo::StatMapped];97 // Maximum total allocated space (bytes)98 Info.usmblks = Info.hblkhd;99 // Space in freed fastbin blocks (bytes)100 Info.fsmblks = Stats[scudo::StatFree];101 // Total allocated space (bytes)102 Info.uordblks = Stats[scudo::StatAllocated];103 // Total free space (bytes)104 Info.fordblks = Info.fsmblks;105 return Info;106}107#endif108 109INTERFACE WEAK void *SCUDO_PREFIX(malloc)(size_t size) {110 void *Ptr = SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Malloc,111 SCUDO_MALLOC_ALIGNMENT);112 reportAllocation(Ptr, size);113 return scudo::setErrnoOnNull(Ptr);114}115 116#if SCUDO_ANDROID117INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(const void *ptr) {118#else119INTERFACE WEAK size_t SCUDO_PREFIX(malloc_usable_size)(void *ptr) {120#endif121 return SCUDO_ALLOCATOR.getUsableSize(ptr);122}123 124INTERFACE WEAK void *SCUDO_PREFIX(memalign)(size_t alignment, size_t size) {125 // Android rounds up the alignment to a power of two if it isn't one.126 if (SCUDO_ANDROID) {127 if (UNLIKELY(!alignment)) {128 alignment = 1U;129 } else {130 if (UNLIKELY(!scudo::isPowerOfTwo(alignment)))131 alignment = scudo::roundUpPowerOfTwo(alignment);132 }133 } else {134 if (UNLIKELY(!scudo::isPowerOfTwo(alignment))) {135 if (SCUDO_ALLOCATOR.canReturnNull()) {136 errno = EINVAL;137 return nullptr;138 }139 scudo::reportAlignmentNotPowerOfTwo(alignment);140 }141 }142 void *Ptr =143 SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Memalign, alignment);144 reportAllocation(Ptr, size);145 return Ptr;146}147 148INTERFACE WEAK int SCUDO_PREFIX(posix_memalign)(void **memptr, size_t alignment,149 size_t size) {150 if (UNLIKELY(scudo::checkPosixMemalignAlignment(alignment))) {151 if (!SCUDO_ALLOCATOR.canReturnNull())152 scudo::reportInvalidPosixMemalignAlignment(alignment);153 return EINVAL;154 }155 void *Ptr =156 SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Memalign, alignment);157 if (UNLIKELY(!Ptr))158 return ENOMEM;159 reportAllocation(Ptr, size);160 161 *memptr = Ptr;162 return 0;163}164 165INTERFACE WEAK void *SCUDO_PREFIX(pvalloc)(size_t size) {166 const scudo::uptr PageSize = scudo::getPageSizeCached();167 if (UNLIKELY(scudo::checkForPvallocOverflow(size, PageSize))) {168 if (SCUDO_ALLOCATOR.canReturnNull()) {169 errno = ENOMEM;170 return nullptr;171 }172 scudo::reportPvallocOverflow(size);173 }174 // pvalloc(0) should allocate one page.175 void *Ptr =176 SCUDO_ALLOCATOR.allocate(size ? scudo::roundUp(size, PageSize) : PageSize,177 scudo::Chunk::Origin::Memalign, PageSize);178 reportAllocation(Ptr, scudo::roundUp(size, PageSize));179 180 return scudo::setErrnoOnNull(Ptr);181}182 183INTERFACE WEAK void *SCUDO_PREFIX(realloc)(void *ptr, size_t size) {184 if (!ptr) {185 void *Ptr = SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Malloc,186 SCUDO_MALLOC_ALIGNMENT);187 reportAllocation(Ptr, size);188 return scudo::setErrnoOnNull(Ptr);189 }190 if (size == 0) {191 reportDeallocation(ptr);192 SCUDO_ALLOCATOR.deallocate(ptr, scudo::Chunk::Origin::Malloc);193 return nullptr;194 }195 196 // Given that the reporting of deallocation and allocation are not atomic, we197 // always pretend the old pointer will be released so that the user doesn't198 // need to worry about the false double-use case from the view of hooks.199 //200 // For example, assume that `realloc` releases the old pointer and allocates a201 // new pointer. Before the reporting of both operations has been done, another202 // thread may get the old pointer from `malloc`. It may be misinterpreted as203 // double-use if it's not handled properly on the hook side.204 reportReallocDeallocation(ptr);205 void *NewPtr = SCUDO_ALLOCATOR.reallocate(ptr, size, SCUDO_MALLOC_ALIGNMENT);206 if (NewPtr != nullptr) {207 // Note that even if NewPtr == ptr, the size has changed. We still need to208 // report the new size.209 reportReallocAllocation(/*OldPtr=*/ptr, NewPtr, size);210 } else {211 // If `realloc` fails, the old pointer is not released. Report the old212 // pointer as allocated again.213 reportReallocAllocation(/*OldPtr=*/ptr, /*NewPtr=*/ptr,214 SCUDO_ALLOCATOR.getAllocSize(ptr));215 }216 217 return scudo::setErrnoOnNull(NewPtr);218}219 220INTERFACE WEAK void *SCUDO_PREFIX(valloc)(size_t size) {221 void *Ptr = SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Memalign,222 scudo::getPageSizeCached());223 reportAllocation(Ptr, size);224 225 return scudo::setErrnoOnNull(Ptr);226}227 228INTERFACE WEAK int SCUDO_PREFIX(malloc_iterate)(229 uintptr_t base, size_t size,230 void (*callback)(uintptr_t base, size_t size, void *arg), void *arg) {231 SCUDO_ALLOCATOR.iterateOverChunks(base, size, callback, arg);232 return 0;233}234 235INTERFACE WEAK void SCUDO_PREFIX(malloc_enable)() { SCUDO_ALLOCATOR.enable(); }236 237INTERFACE WEAK void SCUDO_PREFIX(malloc_disable)() {238 SCUDO_ALLOCATOR.disable();239}240 241void SCUDO_PREFIX(malloc_postinit)() {242 SCUDO_ALLOCATOR.initGwpAsan();243 pthread_atfork(SCUDO_PREFIX(malloc_disable), SCUDO_PREFIX(malloc_enable),244 SCUDO_PREFIX(malloc_enable));245}246 247INTERFACE WEAK int SCUDO_PREFIX(mallopt)(int param, int value) {248 if (param == M_DECAY_TIME) {249 if (SCUDO_ANDROID) {250 // Before changing the interval, reset the memory usage status by doing a251 // M_PURGE call so that we can minimize the impact of any unreleased pages252 // introduced by interval transition.253 SCUDO_ALLOCATOR.releaseToOS(scudo::ReleaseToOS::Force);254 255 // The values allowed on Android are {-1, 0, 1}. "1" means the longest256 // interval.257 CHECK(value >= -1 && value <= 1);258 if (value == 1)259 value = INT32_MAX;260 }261 262 SCUDO_ALLOCATOR.setOption(scudo::Option::ReleaseInterval,263 static_cast<scudo::sptr>(value));264 return 1;265 } else if (param == M_PURGE) {266 SCUDO_ALLOCATOR.releaseToOS(scudo::ReleaseToOS::Force);267 return 1;268 } else if (param == M_PURGE_ALL) {269 SCUDO_ALLOCATOR.releaseToOS(scudo::ReleaseToOS::ForceAll);270 return 1;271 } else if (param == M_LOG_STATS) {272 SCUDO_ALLOCATOR.printStats();273 SCUDO_ALLOCATOR.printFragmentationInfo();274 return 1;275 } else {276 scudo::Option option;277 switch (param) {278 case M_MEMTAG_TUNING:279 option = scudo::Option::MemtagTuning;280 break;281 case M_THREAD_DISABLE_MEM_INIT:282 option = scudo::Option::ThreadDisableMemInit;283 break;284 case M_CACHE_COUNT_MAX:285 option = scudo::Option::MaxCacheEntriesCount;286 break;287 case M_CACHE_SIZE_MAX:288 option = scudo::Option::MaxCacheEntrySize;289 break;290 case M_TSDS_COUNT_MAX:291 option = scudo::Option::MaxTSDsCount;292 break;293 default:294 return 0;295 }296 return SCUDO_ALLOCATOR.setOption(option, static_cast<scudo::sptr>(value));297 }298}299 300INTERFACE WEAK void *SCUDO_PREFIX(aligned_alloc)(size_t alignment,301 size_t size) {302 if (UNLIKELY(scudo::checkAlignedAllocAlignmentAndSize(alignment, size))) {303 if (SCUDO_ALLOCATOR.canReturnNull()) {304 errno = EINVAL;305 return nullptr;306 }307 scudo::reportInvalidAlignedAllocAlignment(alignment, size);308 }309 310 void *Ptr =311 SCUDO_ALLOCATOR.allocate(size, scudo::Chunk::Origin::Malloc, alignment);312 reportAllocation(Ptr, size);313 314 return scudo::setErrnoOnNull(Ptr);315}316 317INTERFACE WEAK int SCUDO_PREFIX(malloc_info)(UNUSED int options, FILE *stream) {318 const scudo::uptr max_size =319 decltype(SCUDO_ALLOCATOR)::PrimaryT::SizeClassMap::MaxSize;320 auto *sizes = static_cast<scudo::uptr *>(321 SCUDO_PREFIX(calloc)(max_size, sizeof(scudo::uptr)));322 auto callback = [](uintptr_t, size_t size, void *arg) {323 auto *sizes = reinterpret_cast<scudo::uptr *>(arg);324 if (size < max_size)325 sizes[size]++;326 };327 328 SCUDO_ALLOCATOR.disable();329 SCUDO_ALLOCATOR.iterateOverChunks(0, -1ul, callback, sizes);330 SCUDO_ALLOCATOR.enable();331 332 fputs("<malloc version=\"scudo-1\">\n", stream);333 for (scudo::uptr i = 0; i != max_size; ++i)334 if (sizes[i])335 fprintf(stream, "<alloc size=\"%zu\" count=\"%zu\"/>\n", i, sizes[i]);336 fputs("</malloc>\n", stream);337 SCUDO_PREFIX(free)(sizes);338 return 0;339}340 341// Disable memory tagging for the heap. The caller must disable memory tag342// checks globally (e.g. by clearing TCF0 on aarch64) before calling this343// function, and may not re-enable them after calling the function.344INTERFACE WEAK void SCUDO_PREFIX(malloc_disable_memory_tagging)() {345 SCUDO_ALLOCATOR.disableMemoryTagging();346}347 348// Sets whether scudo records stack traces and other metadata for allocations349// and deallocations. This function only has an effect if the allocator and350// hardware support memory tagging.351INTERFACE WEAK void352SCUDO_PREFIX(malloc_set_track_allocation_stacks)(int track) {353 SCUDO_ALLOCATOR.setTrackAllocationStacks(track);354}355 356// Sets whether scudo zero-initializes all allocated memory.357INTERFACE WEAK void SCUDO_PREFIX(malloc_set_zero_contents)(int zero_contents) {358 SCUDO_ALLOCATOR.setFillContents(zero_contents ? scudo::ZeroFill359 : scudo::NoFill);360}361 362// Sets whether scudo pattern-initializes all allocated memory.363INTERFACE WEAK void364SCUDO_PREFIX(malloc_set_pattern_fill_contents)(int pattern_fill_contents) {365 SCUDO_ALLOCATOR.setFillContents(366 pattern_fill_contents ? scudo::PatternOrZeroFill : scudo::NoFill);367}368 369// Sets whether scudo adds a small amount of slack at the end of large370// allocations, before the guard page. This can be enabled to work around buggy371// applications that read a few bytes past the end of their allocation.372INTERFACE WEAK void373SCUDO_PREFIX(malloc_set_add_large_allocation_slack)(int add_slack) {374 SCUDO_ALLOCATOR.setAddLargeAllocationSlack(add_slack);375}376 377} // extern "C"378