1818 lines · c
1//===-- combined.h ----------------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#ifndef SCUDO_COMBINED_H_10#define SCUDO_COMBINED_H_11 12#include "allocator_config_wrapper.h"13#include "atomic_helpers.h"14#include "chunk.h"15#include "common.h"16#include "flags.h"17#include "flags_parser.h"18#include "mem_map.h"19#include "memtag.h"20#include "mutex.h"21#include "options.h"22#include "quarantine.h"23#include "report.h"24#include "secondary.h"25#include "size_class_allocator.h"26#include "stack_depot.h"27#include "string_utils.h"28#include "tracing.h"29#include "tsd.h"30 31#include "scudo/interface.h"32 33#ifdef GWP_ASAN_HOOKS34#include "gwp_asan/guarded_pool_allocator.h"35#include "gwp_asan/optional/backtrace.h"36#include "gwp_asan/optional/segv_handler.h"37#endif // GWP_ASAN_HOOKS38 39extern "C" inline void EmptyCallback() {}40 41#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE42// This function is not part of the NDK so it does not appear in any public43// header files. We only declare/use it when targeting the platform.44extern "C" size_t android_unsafe_frame_pointer_chase(scudo::uptr *buf,45 size_t num_entries);46#endif47 48namespace scudo {49 50template <class Config, void (*PostInitCallback)(void) = EmptyCallback>51class Allocator {52public:53 using AllocatorConfig = BaseConfig<Config>;54 using PrimaryT =55 typename AllocatorConfig::template PrimaryT<PrimaryConfig<Config>>;56 using SecondaryT =57 typename AllocatorConfig::template SecondaryT<SecondaryConfig<Config>>;58 using SizeClassAllocatorT = typename PrimaryT::SizeClassAllocatorT;59 typedef Allocator<Config, PostInitCallback> ThisT;60 typedef typename AllocatorConfig::template TSDRegistryT<ThisT> TSDRegistryT;61 62 void callPostInitCallback() {63 pthread_once(&PostInitNonce, PostInitCallback);64 }65 66 struct QuarantineCallback {67 explicit QuarantineCallback(ThisT &Instance,68 SizeClassAllocatorT &SizeClassAllocator)69 : Allocator(Instance), SizeClassAllocator(SizeClassAllocator) {}70 71 // Chunk recycling function, returns a quarantined chunk to the backend,72 // first making sure it hasn't been tampered with.73 void recycle(void *Ptr) {74 Chunk::UnpackedHeader Header;75 Chunk::loadHeader(Allocator.Cookie, Ptr, &Header);76 if (UNLIKELY(Header.State != Chunk::State::Quarantined))77 reportInvalidChunkState(AllocatorAction::Recycling, Ptr);78 79 Header.State = Chunk::State::Available;80 Chunk::storeHeader(Allocator.Cookie, Ptr, &Header);81 82 if (allocatorSupportsMemoryTagging<AllocatorConfig>())83 Ptr = untagPointer(Ptr);84 void *BlockBegin = Allocator::getBlockBegin(Ptr, &Header);85 SizeClassAllocator.deallocate(Header.ClassId, BlockBegin);86 }87 88 // We take a shortcut when allocating a quarantine batch by working with the89 // appropriate class ID instead of using Size. The compiler should optimize90 // the class ID computation and work with the associated cache directly.91 void *allocate(UNUSED uptr Size) {92 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(93 sizeof(QuarantineBatch) + Chunk::getHeaderSize());94 void *Ptr = SizeClassAllocator.allocate(QuarantineClassId);95 // Quarantine batch allocation failure is fatal.96 if (UNLIKELY(!Ptr))97 reportOutOfMemory(SizeClassMap::getSizeByClassId(QuarantineClassId));98 99 Ptr = reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) +100 Chunk::getHeaderSize());101 Chunk::UnpackedHeader Header = {};102 Header.ClassId = QuarantineClassId & Chunk::ClassIdMask;103 Header.SizeOrUnusedBytes = sizeof(QuarantineBatch);104 Header.State = Chunk::State::Quarantined;105 Chunk::storeHeader(Allocator.Cookie, Ptr, &Header);106 107 // Reset tag to 0 as this chunk may have been previously used for a tagged108 // user allocation.109 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(110 Allocator.Primary.Options.load())))111 storeTags(reinterpret_cast<uptr>(Ptr),112 reinterpret_cast<uptr>(Ptr) + sizeof(QuarantineBatch));113 114 return Ptr;115 }116 117 void deallocate(void *Ptr) {118 const uptr QuarantineClassId = SizeClassMap::getClassIdBySize(119 sizeof(QuarantineBatch) + Chunk::getHeaderSize());120 Chunk::UnpackedHeader Header;121 Chunk::loadHeader(Allocator.Cookie, Ptr, &Header);122 123 if (UNLIKELY(Header.State != Chunk::State::Quarantined))124 reportInvalidChunkState(AllocatorAction::Deallocating, Ptr);125 DCHECK_EQ(Header.ClassId, QuarantineClassId);126 DCHECK_EQ(Header.Offset, 0);127 DCHECK_EQ(Header.SizeOrUnusedBytes, sizeof(QuarantineBatch));128 129 Header.State = Chunk::State::Available;130 Chunk::storeHeader(Allocator.Cookie, Ptr, &Header);131 SizeClassAllocator.deallocate(132 QuarantineClassId,133 reinterpret_cast<void *>(reinterpret_cast<uptr>(Ptr) -134 Chunk::getHeaderSize()));135 }136 137 private:138 ThisT &Allocator;139 SizeClassAllocatorT &SizeClassAllocator;140 };141 142 typedef GlobalQuarantine<QuarantineCallback, void> QuarantineT;143 typedef typename QuarantineT::CacheT QuarantineCacheT;144 145 void init() {146 // Make sure that the page size is initialized if it's not a constant.147 CHECK_NE(getPageSizeCached(), 0U);148 149 performSanityChecks();150 151 // Check if hardware CRC32 is supported in the binary and by the platform,152 // if so, opt for the CRC32 hardware version of the checksum.153 if (&computeHardwareCRC32 && hasHardwareCRC32())154 HashAlgorithm = Checksum::HardwareCRC32;155 156 if (UNLIKELY(!getRandom(&Cookie, sizeof(Cookie))))157 Cookie = static_cast<u32>(getMonotonicTime() ^158 (reinterpret_cast<uptr>(this) >> 4));159 160 initFlags();161 reportUnrecognizedFlags();162 163 // Store some flags locally.164 if (getFlags()->may_return_null)165 Primary.Options.set(OptionBit::MayReturnNull);166 if (getFlags()->zero_contents)167 Primary.Options.setFillContentsMode(ZeroFill);168 else if (getFlags()->pattern_fill_contents)169 Primary.Options.setFillContentsMode(PatternOrZeroFill);170 if (getFlags()->dealloc_type_mismatch)171 Primary.Options.set(OptionBit::DeallocTypeMismatch);172 if (getFlags()->delete_size_mismatch)173 Primary.Options.set(OptionBit::DeleteSizeMismatch);174 if (systemSupportsMemoryTagging())175 Primary.Options.set(OptionBit::UseMemoryTagging);176 177 QuarantineMaxChunkSize =178 static_cast<u32>(getFlags()->quarantine_max_chunk_size);179 180 Stats.init();181 // TODO(chiahungduan): Given that we support setting the default value in182 // the PrimaryConfig and CacheConfig, consider to deprecate the use of183 // `release_to_os_interval_ms` flag.184 const s32 ReleaseToOsIntervalMs = getFlags()->release_to_os_interval_ms;185 Primary.init(ReleaseToOsIntervalMs);186 Secondary.init(&Stats, ReleaseToOsIntervalMs);187 if (!AllocatorConfig::getQuarantineDisabled()) {188 Quarantine.init(189 static_cast<uptr>(getFlags()->quarantine_size_kb << 10),190 static_cast<uptr>(getFlags()->thread_local_quarantine_size_kb << 10));191 }192 }193 194 void enableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {195 AllocationRingBuffer *RB = getRingBuffer();196 if (RB)197 RB->Depot->enable();198 RingBufferInitLock.unlock();199 }200 201 void disableRingBuffer() NO_THREAD_SAFETY_ANALYSIS {202 RingBufferInitLock.lock();203 AllocationRingBuffer *RB = getRingBuffer();204 if (RB)205 RB->Depot->disable();206 }207 208 // Initialize the embedded GWP-ASan instance. Requires the main allocator to209 // be functional, best called from PostInitCallback.210 void initGwpAsan() {211#ifdef GWP_ASAN_HOOKS212 gwp_asan::options::Options Opt;213 Opt.Enabled = getFlags()->GWP_ASAN_Enabled;214 Opt.MaxSimultaneousAllocations =215 getFlags()->GWP_ASAN_MaxSimultaneousAllocations;216 Opt.SampleRate = getFlags()->GWP_ASAN_SampleRate;217 Opt.InstallSignalHandlers = getFlags()->GWP_ASAN_InstallSignalHandlers;218 Opt.Recoverable = getFlags()->GWP_ASAN_Recoverable;219 // Embedded GWP-ASan is locked through the Scudo atfork handler (via220 // Allocator::disable calling GWPASan.disable). Disable GWP-ASan's atfork221 // handler.222 Opt.InstallForkHandlers = false;223 Opt.Backtrace = gwp_asan::backtrace::getBacktraceFunction();224 GuardedAlloc.init(Opt);225 226 if (Opt.InstallSignalHandlers)227 gwp_asan::segv_handler::installSignalHandlers(228 &GuardedAlloc, Printf,229 gwp_asan::backtrace::getPrintBacktraceFunction(),230 gwp_asan::backtrace::getSegvBacktraceFunction(),231 Opt.Recoverable);232 233 GuardedAllocSlotSize =234 GuardedAlloc.getAllocatorState()->maximumAllocationSize();235 Stats.add(StatFree, static_cast<uptr>(Opt.MaxSimultaneousAllocations) *236 GuardedAllocSlotSize);237#endif // GWP_ASAN_HOOKS238 }239 240#ifdef GWP_ASAN_HOOKS241 const gwp_asan::AllocationMetadata *getGwpAsanAllocationMetadata() {242 return GuardedAlloc.getMetadataRegion();243 }244 245 const gwp_asan::AllocatorState *getGwpAsanAllocatorState() {246 return GuardedAlloc.getAllocatorState();247 }248#endif // GWP_ASAN_HOOKS249 250 ALWAYS_INLINE void initThreadMaybe(bool MinimalInit = false) {251 TSDRegistry.initThreadMaybe(this, MinimalInit);252 }253 254 void unmapTestOnly() {255 unmapRingBuffer();256 TSDRegistry.unmapTestOnly(this);257 Primary.unmapTestOnly();258 Secondary.unmapTestOnly();259#ifdef GWP_ASAN_HOOKS260 if (getFlags()->GWP_ASAN_InstallSignalHandlers)261 gwp_asan::segv_handler::uninstallSignalHandlers();262 GuardedAlloc.uninitTestOnly();263#endif // GWP_ASAN_HOOKS264 }265 266 TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }267 QuarantineT *getQuarantine() { return &Quarantine; }268 269 // The Cache must be provided zero-initialized.270 void initAllocator(SizeClassAllocatorT *SizeClassAllocator) {271 SizeClassAllocator->init(&Stats, &Primary);272 }273 274 // Release the resources used by a TSD, which involves:275 // - draining the local quarantine cache to the global quarantine;276 // - releasing the cached pointers back to the Primary;277 // - unlinking the local stats from the global ones (destroying the cache does278 // the last two items).279 void commitBack(TSD<ThisT> *TSD) {280 TSD->assertLocked(/*BypassCheck=*/true);281 if (!AllocatorConfig::getQuarantineDisabled()) {282 Quarantine.drain(&TSD->getQuarantineCache(),283 QuarantineCallback(*this, TSD->getSizeClassAllocator()));284 }285 TSD->getSizeClassAllocator().destroy(&Stats);286 }287 288 void drainCache(TSD<ThisT> *TSD) {289 TSD->assertLocked(/*BypassCheck=*/true);290 if (!AllocatorConfig::getQuarantineDisabled()) {291 Quarantine.drainAndRecycle(292 &TSD->getQuarantineCache(),293 QuarantineCallback(*this, TSD->getSizeClassAllocator()));294 }295 TSD->getSizeClassAllocator().drain();296 }297 void drainCaches() { TSDRegistry.drainCaches(this); }298 299 ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) {300 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())301 return Ptr;302 auto UntaggedPtr = untagPointer(Ptr);303 if (UntaggedPtr != Ptr)304 return UntaggedPtr;305 // Secondary, or pointer allocated while memory tagging is unsupported or306 // disabled. The tag mismatch is okay in the latter case because tags will307 // not be checked.308 return addHeaderTag(Ptr);309 }310 311 ALWAYS_INLINE uptr addHeaderTag(uptr Ptr) {312 if (!allocatorSupportsMemoryTagging<AllocatorConfig>())313 return Ptr;314 return addFixedTag(Ptr, 2);315 }316 317 ALWAYS_INLINE void *addHeaderTag(void *Ptr) {318 return reinterpret_cast<void *>(addHeaderTag(reinterpret_cast<uptr>(Ptr)));319 }320 321 NOINLINE u32 collectStackTrace(UNUSED StackDepot *Depot) {322#ifdef HAVE_ANDROID_UNSAFE_FRAME_POINTER_CHASE323 // Discard collectStackTrace() frame and allocator function frame.324 constexpr uptr DiscardFrames = 2;325 uptr Stack[MaxTraceSize + DiscardFrames];326 uptr Size =327 android_unsafe_frame_pointer_chase(Stack, MaxTraceSize + DiscardFrames);328 Size = Min<uptr>(Size, MaxTraceSize + DiscardFrames);329 return Depot->insert(Stack + Min<uptr>(DiscardFrames, Size), Stack + Size);330#else331 return 0;332#endif333 }334 335 uptr computeOddEvenMaskForPointerMaybe(const Options &Options, uptr Ptr,336 uptr ClassId) {337 if (!Options.get(OptionBit::UseOddEvenTags))338 return 0;339 340 // If a chunk's tag is odd, we want the tags of the surrounding blocks to be341 // even, and vice versa. Blocks are laid out Size bytes apart, and adding342 // Size to Ptr will flip the least significant set bit of Size in Ptr, so343 // that bit will have the pattern 010101... for consecutive blocks, which we344 // can use to determine which tag mask to use.345 return 0x5555U << ((Ptr >> SizeClassMap::getSizeLSBByClassId(ClassId)) & 1);346 }347 348 NOINLINE void *allocate(uptr Size, Chunk::Origin Origin,349 uptr Alignment = MinAlignment,350 bool ZeroContents = false) NO_THREAD_SAFETY_ANALYSIS {351 initThreadMaybe();352 353 const Options Options = Primary.Options.load();354 if (UNLIKELY(Alignment > MaxAlignment)) {355 if (Options.get(OptionBit::MayReturnNull))356 return nullptr;357 reportAlignmentTooBig(Alignment, MaxAlignment);358 }359 if (Alignment < MinAlignment)360 Alignment = MinAlignment;361 362#ifdef GWP_ASAN_HOOKS363 if (UNLIKELY(GuardedAlloc.shouldSample())) {364 if (void *Ptr = GuardedAlloc.allocate(Size, Alignment)) {365 Stats.lock();366 Stats.add(StatAllocated, GuardedAllocSlotSize);367 Stats.sub(StatFree, GuardedAllocSlotSize);368 Stats.unlock();369 return Ptr;370 }371 }372#endif // GWP_ASAN_HOOKS373 374 const FillContentsMode FillContents = ZeroContents ? ZeroFill375 : TSDRegistry.getDisableMemInit()376 ? NoFill377 : Options.getFillContentsMode();378 379 // If the requested size happens to be 0 (more common than you might think),380 // allocate MinAlignment bytes on top of the header. Then add the extra381 // bytes required to fulfill the alignment requirements: we allocate enough382 // to be sure that there will be an address in the block that will satisfy383 // the alignment.384 const uptr NeededSize =385 roundUp(Size, MinAlignment) +386 ((Alignment > MinAlignment) ? Alignment : Chunk::getHeaderSize());387 388 // Takes care of extravagantly large sizes as well as integer overflows.389 static_assert(MaxAllowedMallocSize < UINTPTR_MAX - MaxAlignment, "");390 if (UNLIKELY(Size >= MaxAllowedMallocSize)) {391 if (Options.get(OptionBit::MayReturnNull))392 return nullptr;393 reportAllocationSizeTooBig(Size, NeededSize, MaxAllowedMallocSize);394 }395 DCHECK_LE(Size, NeededSize);396 397 void *Block = nullptr;398 uptr ClassId = 0;399 uptr SecondaryBlockEnd = 0;400 if (LIKELY(PrimaryT::canAllocate(NeededSize))) {401 ClassId = SizeClassMap::getClassIdBySize(NeededSize);402 DCHECK_NE(ClassId, 0U);403 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);404 Block = TSD->getSizeClassAllocator().allocate(ClassId);405 // If the allocation failed, retry in each successively larger class until406 // it fits. If it fails to fit in the largest class, fallback to the407 // Secondary.408 if (UNLIKELY(!Block)) {409 while (ClassId < SizeClassMap::LargestClassId && !Block)410 Block = TSD->getSizeClassAllocator().allocate(++ClassId);411 if (!Block)412 ClassId = 0;413 }414 }415 if (UNLIKELY(ClassId == 0)) {416 Block = Secondary.allocate(Options, Size, Alignment, &SecondaryBlockEnd,417 FillContents);418 }419 420 if (UNLIKELY(!Block)) {421 if (Options.get(OptionBit::MayReturnNull))422 return nullptr;423 printStats();424 reportOutOfMemory(NeededSize);425 }426 427 const uptr UserPtr = roundUp(428 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize(), Alignment);429 const uptr SizeOrUnusedBytes =430 ClassId ? Size : SecondaryBlockEnd - (UserPtr + Size);431 432 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {433 return initChunk(ClassId, Origin, Block, UserPtr, SizeOrUnusedBytes,434 FillContents);435 }436 437 return initChunkWithMemoryTagging(ClassId, Origin, Block, UserPtr, Size,438 SizeOrUnusedBytes, FillContents);439 }440 441 NOINLINE void deallocate(void *Ptr, Chunk::Origin Origin, uptr DeleteSize = 0,442 UNUSED uptr Alignment = MinAlignment) {443 if (UNLIKELY(!Ptr))444 return;445 446 // For a deallocation, we only ensure minimal initialization, meaning thread447 // local data will be left uninitialized for now (when using ELF TLS). The448 // fallback cache will be used instead. This is a workaround for a situation449 // where the only heap operation performed in a thread would be a free past450 // the TLS destructors, ending up in initialized thread specific data never451 // being destroyed properly. Any other heap operation will do a full init.452 initThreadMaybe(/*MinimalInit=*/true);453 454#ifdef GWP_ASAN_HOOKS455 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr))) {456 GuardedAlloc.deallocate(Ptr);457 Stats.lock();458 Stats.add(StatFree, GuardedAllocSlotSize);459 Stats.sub(StatAllocated, GuardedAllocSlotSize);460 Stats.unlock();461 return;462 }463#endif // GWP_ASAN_HOOKS464 465 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment)))466 reportMisalignedPointer(AllocatorAction::Deallocating, Ptr);467 468 void *TaggedPtr = Ptr;469 Ptr = getHeaderTaggedPointer(Ptr);470 471 Chunk::UnpackedHeader Header;472 Chunk::loadHeader(Cookie, Ptr, &Header);473 474 if (UNLIKELY(Header.State != Chunk::State::Allocated))475 reportInvalidChunkState(AllocatorAction::Deallocating, Ptr);476 477 const Options Options = Primary.Options.load();478 if (Options.get(OptionBit::DeallocTypeMismatch)) {479 if (UNLIKELY(Header.OriginOrWasZeroed != Origin)) {480 // With the exception of memalign'd chunks, that can be still be free'd.481 if (Header.OriginOrWasZeroed != Chunk::Origin::Memalign ||482 Origin != Chunk::Origin::Malloc)483 reportDeallocTypeMismatch(AllocatorAction::Deallocating, Ptr,484 Header.OriginOrWasZeroed, Origin);485 }486 }487 488 const uptr Size = getSize(Ptr, &Header);489 if (DeleteSize && Options.get(OptionBit::DeleteSizeMismatch)) {490 if (UNLIKELY(DeleteSize != Size))491 reportDeleteSizeMismatch(Ptr, DeleteSize, Size);492 }493 494 quarantineOrDeallocateChunk(Options, TaggedPtr, &Header, Size);495 }496 497 void *reallocate(void *OldPtr, uptr NewSize, uptr Alignment = MinAlignment) {498 initThreadMaybe();499 500 const Options Options = Primary.Options.load();501 if (UNLIKELY(NewSize >= MaxAllowedMallocSize)) {502 if (Options.get(OptionBit::MayReturnNull))503 return nullptr;504 reportAllocationSizeTooBig(NewSize, 0, MaxAllowedMallocSize);505 }506 507 // The following cases are handled by the C wrappers.508 DCHECK_NE(OldPtr, nullptr);509 DCHECK_NE(NewSize, 0);510 511#ifdef GWP_ASAN_HOOKS512 if (UNLIKELY(GuardedAlloc.pointerIsMine(OldPtr))) {513 uptr OldSize = GuardedAlloc.getSize(OldPtr);514 void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment);515 if (NewPtr)516 memcpy(NewPtr, OldPtr, (NewSize < OldSize) ? NewSize : OldSize);517 GuardedAlloc.deallocate(OldPtr);518 Stats.lock();519 Stats.add(StatFree, GuardedAllocSlotSize);520 Stats.sub(StatAllocated, GuardedAllocSlotSize);521 Stats.unlock();522 return NewPtr;523 }524#endif // GWP_ASAN_HOOKS525 526 void *OldTaggedPtr = OldPtr;527 OldPtr = getHeaderTaggedPointer(OldPtr);528 529 if (UNLIKELY(!isAligned(reinterpret_cast<uptr>(OldPtr), MinAlignment)))530 reportMisalignedPointer(AllocatorAction::Reallocating, OldPtr);531 532 Chunk::UnpackedHeader Header;533 Chunk::loadHeader(Cookie, OldPtr, &Header);534 535 if (UNLIKELY(Header.State != Chunk::State::Allocated))536 reportInvalidChunkState(AllocatorAction::Reallocating, OldPtr);537 538 // Pointer has to be allocated with a malloc-type function. Some539 // applications think that it is OK to realloc a memalign'ed pointer, which540 // will trigger this check. It really isn't.541 if (Options.get(OptionBit::DeallocTypeMismatch)) {542 if (UNLIKELY(Header.OriginOrWasZeroed != Chunk::Origin::Malloc))543 reportDeallocTypeMismatch(AllocatorAction::Reallocating, OldPtr,544 Header.OriginOrWasZeroed,545 Chunk::Origin::Malloc);546 }547 548 void *BlockBegin = getBlockBegin(OldTaggedPtr, &Header);549 uptr BlockEnd;550 uptr OldSize;551 const uptr ClassId = Header.ClassId;552 if (LIKELY(ClassId)) {553 BlockEnd = reinterpret_cast<uptr>(BlockBegin) +554 SizeClassMap::getSizeByClassId(ClassId);555 OldSize = Header.SizeOrUnusedBytes;556 } else {557 BlockEnd = SecondaryT::getBlockEnd(BlockBegin);558 OldSize = BlockEnd - (reinterpret_cast<uptr>(OldTaggedPtr) +559 Header.SizeOrUnusedBytes);560 }561 // If the new chunk still fits in the previously allocated block (with a562 // reasonable delta), we just keep the old block, and update the chunk563 // header to reflect the size change.564 if (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize <= BlockEnd) {565 if (NewSize > OldSize || (OldSize - NewSize) < getPageSizeCached()) {566 // If we have reduced the size, set the extra bytes to the fill value567 // so that we are ready to grow it again in the future.568 if (NewSize < OldSize) {569 const FillContentsMode FillContents =570 TSDRegistry.getDisableMemInit() ? NoFill571 : Options.getFillContentsMode();572 if (FillContents != NoFill) {573 memset(reinterpret_cast<char *>(OldTaggedPtr) + NewSize,574 FillContents == ZeroFill ? 0 : PatternFillByte,575 OldSize - NewSize);576 }577 }578 579 Header.SizeOrUnusedBytes =580 (ClassId ? NewSize581 : BlockEnd -582 (reinterpret_cast<uptr>(OldTaggedPtr) + NewSize)) &583 Chunk::SizeOrUnusedBytesMask;584 Chunk::storeHeader(Cookie, OldPtr, &Header);585 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options))) {586 if (ClassId) {587 resizeTaggedChunk(reinterpret_cast<uptr>(OldTaggedPtr) + OldSize,588 reinterpret_cast<uptr>(OldTaggedPtr) + NewSize,589 NewSize, untagPointer(BlockEnd));590 storePrimaryAllocationStackMaybe(Options, OldPtr);591 } else {592 storeSecondaryAllocationStackMaybe(Options, OldPtr, NewSize);593 }594 }595 return OldTaggedPtr;596 }597 }598 599 // Otherwise we allocate a new one, and deallocate the old one. Some600 // allocators will allocate an even larger chunk (by a fixed factor) to601 // allow for potential further in-place realloc. The gains of such a trick602 // are currently unclear.603 void *NewPtr = allocate(NewSize, Chunk::Origin::Malloc, Alignment);604 if (LIKELY(NewPtr)) {605 memcpy(NewPtr, OldTaggedPtr, Min(NewSize, OldSize));606 quarantineOrDeallocateChunk(Options, OldTaggedPtr, &Header, OldSize);607 }608 return NewPtr;609 }610 611 // TODO(kostyak): disable() is currently best-effort. There are some small612 // windows of time when an allocation could still succeed after613 // this function finishes. We will revisit that later.614 void disable() NO_THREAD_SAFETY_ANALYSIS {615 initThreadMaybe();616#ifdef GWP_ASAN_HOOKS617 GuardedAlloc.disable();618#endif619 TSDRegistry.disable();620 Stats.disable();621 if (!AllocatorConfig::getQuarantineDisabled())622 Quarantine.disable();623 Primary.disable();624 Secondary.disable();625 disableRingBuffer();626 }627 628 void enable() NO_THREAD_SAFETY_ANALYSIS {629 initThreadMaybe();630 enableRingBuffer();631 Secondary.enable();632 Primary.enable();633 if (!AllocatorConfig::getQuarantineDisabled())634 Quarantine.enable();635 Stats.enable();636 TSDRegistry.enable();637#ifdef GWP_ASAN_HOOKS638 GuardedAlloc.enable();639#endif640 }641 642 // The function returns the amount of bytes required to store the statistics,643 // which might be larger than the amount of bytes provided. Note that the644 // statistics buffer is not necessarily constant between calls to this645 // function. This can be called with a null buffer or zero size for buffer646 // sizing purposes.647 uptr getStats(char *Buffer, uptr Size) {648 ScopedString Str;649 const uptr Length = getStats(&Str) + 1;650 if (Length < Size)651 Size = Length;652 if (Buffer && Size) {653 memcpy(Buffer, Str.data(), Size);654 Buffer[Size - 1] = '\0';655 }656 return Length;657 }658 659 void printStats() {660 ScopedString Str;661 getStats(&Str);662 Str.output();663 }664 665 void printFragmentationInfo() {666 ScopedString Str;667 Primary.getFragmentationInfo(&Str);668 // Secondary allocator dumps the fragmentation data in getStats().669 Str.output();670 }671 672 void releaseToOS(ReleaseToOS ReleaseType) {673 initThreadMaybe();674 SCUDO_SCOPED_TRACE(GetReleaseToOSTraceName(ReleaseType));675 if (ReleaseType == ReleaseToOS::ForceAll)676 drainCaches();677 Primary.releaseToOS(ReleaseType);678 Secondary.releaseToOS(ReleaseType);679 }680 681 // Iterate over all chunks and call a callback for all busy chunks located682 // within the provided memory range. Said callback must not use this allocator683 // or a deadlock can ensue. This fits Android's malloc_iterate() needs.684 void iterateOverChunks(uptr Base, uptr Size, iterate_callback Callback,685 void *Arg) {686 initThreadMaybe();687 if (archSupportsMemoryTagging())688 Base = untagPointer(Base);689 const uptr From = Base;690 const uptr To = Base + Size;691 const Options Options = Primary.Options.load();692 bool MayHaveTaggedPrimary = useMemoryTagging<AllocatorConfig>(Options);693 auto Lambda = [this, From, To, MayHaveTaggedPrimary, Callback,694 Arg](uptr Block) {695 if (Block < From || Block >= To)696 return;697 uptr Chunk;698 Chunk::UnpackedHeader Header;699 if (UNLIKELY(MayHaveTaggedPrimary)) {700 // A chunk header can either have a zero tag (tagged primary) or the701 // header tag (secondary, or untagged primary). We don't know which so702 // try both.703 ScopedDisableMemoryTagChecks x;704 if (!getChunkFromBlock(Block, &Chunk, &Header) &&705 !getChunkFromBlock(addHeaderTag(Block), &Chunk, &Header))706 return;707 } else if (!getChunkFromBlock(addHeaderTag(Block), &Chunk, &Header)) {708 return;709 }710 711 if (Header.State != Chunk::State::Allocated)712 return;713 714 uptr TaggedChunk = Chunk;715 if (allocatorSupportsMemoryTagging<AllocatorConfig>())716 TaggedChunk = untagPointer(TaggedChunk);717 uptr Size;718 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Primary.Options.load()))) {719 TaggedChunk = loadTag(Chunk);720 Size = getSize(reinterpret_cast<void *>(Chunk), &Header);721 } else if (AllocatorConfig::getExactUsableSize()) {722 Size = getSize(reinterpret_cast<void *>(Chunk), &Header);723 } else {724 Size = getUsableSize(reinterpret_cast<void *>(Chunk), &Header);725 }726 Callback(TaggedChunk, Size, Arg);727 };728 Primary.iterateOverBlocks(Lambda);729 Secondary.iterateOverBlocks(Lambda);730#ifdef GWP_ASAN_HOOKS731 GuardedAlloc.iterate(reinterpret_cast<void *>(Base), Size, Callback, Arg);732#endif733 }734 735 bool canReturnNull() {736 initThreadMaybe();737 return Primary.Options.load().get(OptionBit::MayReturnNull);738 }739 740 bool setOption(Option O, sptr Value) {741 initThreadMaybe();742 if (O == Option::MemtagTuning) {743 // Enabling odd/even tags involves a tradeoff between use-after-free744 // detection and buffer overflow detection. Odd/even tags make it more745 // likely for buffer overflows to be detected by increasing the size of746 // the guaranteed "red zone" around the allocation, but on the other hand747 // use-after-free is less likely to be detected because the tag space for748 // any particular chunk is cut in half. Therefore we use this tuning749 // setting to control whether odd/even tags are enabled.750 if (Value == M_MEMTAG_TUNING_BUFFER_OVERFLOW)751 Primary.Options.set(OptionBit::UseOddEvenTags);752 else if (Value == M_MEMTAG_TUNING_UAF)753 Primary.Options.clear(OptionBit::UseOddEvenTags);754 return true;755 } else {756 // We leave it to the various sub-components to decide whether or not they757 // want to handle the option, but we do not want to short-circuit758 // execution if one of the setOption was to return false.759 const bool PrimaryResult = Primary.setOption(O, Value);760 const bool SecondaryResult = Secondary.setOption(O, Value);761 const bool RegistryResult = TSDRegistry.setOption(O, Value);762 return PrimaryResult && SecondaryResult && RegistryResult;763 }764 return false;765 }766 767 ALWAYS_INLINE uptr getUsableSize(const void *Ptr,768 Chunk::UnpackedHeader *Header) {769 void *BlockBegin = getBlockBegin(Ptr, Header);770 if (LIKELY(Header->ClassId)) {771 return SizeClassMap::getSizeByClassId(Header->ClassId) -772 (reinterpret_cast<uptr>(Ptr) - reinterpret_cast<uptr>(BlockBegin));773 }774 775 uptr UntaggedPtr = reinterpret_cast<uptr>(Ptr);776 if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {777 UntaggedPtr = untagPointer(UntaggedPtr);778 BlockBegin = untagPointer(BlockBegin);779 }780 return SecondaryT::getBlockEnd(BlockBegin) - UntaggedPtr;781 }782 783 // Return the usable size for a given chunk. If MTE is enabled or if the784 // ExactUsableSize config parameter is true, we report the exact size of785 // the original allocation size. Otherwise, we will return the total786 // actual usable size.787 uptr getUsableSize(const void *Ptr) {788 if (UNLIKELY(!Ptr))789 return 0;790 791 if (AllocatorConfig::getExactUsableSize() ||792 UNLIKELY(useMemoryTagging<AllocatorConfig>(Primary.Options.load())))793 return getAllocSize(Ptr);794 795 initThreadMaybe();796 797#ifdef GWP_ASAN_HOOKS798 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))799 return GuardedAlloc.getSize(Ptr);800#endif // GWP_ASAN_HOOKS801 802 Ptr = getHeaderTaggedPointer(const_cast<void *>(Ptr));803 Chunk::UnpackedHeader Header;804 Chunk::loadHeader(Cookie, Ptr, &Header);805 806 // Getting the alloc size of a chunk only makes sense if it's allocated.807 if (UNLIKELY(Header.State != Chunk::State::Allocated))808 reportInvalidChunkState(AllocatorAction::Sizing, Ptr);809 810 return getUsableSize(Ptr, &Header);811 }812 813 uptr getAllocSize(const void *Ptr) {814 initThreadMaybe();815 816#ifdef GWP_ASAN_HOOKS817 if (UNLIKELY(GuardedAlloc.pointerIsMine(Ptr)))818 return GuardedAlloc.getSize(Ptr);819#endif // GWP_ASAN_HOOKS820 821 Ptr = getHeaderTaggedPointer(const_cast<void *>(Ptr));822 Chunk::UnpackedHeader Header;823 Chunk::loadHeader(Cookie, Ptr, &Header);824 825 // Getting the alloc size of a chunk only makes sense if it's allocated.826 if (UNLIKELY(Header.State != Chunk::State::Allocated))827 reportInvalidChunkState(AllocatorAction::Sizing, Ptr);828 829 return getSize(Ptr, &Header);830 }831 832 void getStats(StatCounters S) {833 initThreadMaybe();834 Stats.get(S);835 }836 837 // Returns true if the pointer provided was allocated by the current838 // allocator instance, which is compliant with tcmalloc's ownership concept.839 // A corrupted chunk will not be reported as owned, which is WAI.840 bool isOwned(const void *Ptr) {841 initThreadMaybe();842 // If the allocation is not owned, the tags could be wrong.843 ScopedDisableMemoryTagChecks x(844 useMemoryTagging<AllocatorConfig>(Primary.Options.load()));845#ifdef GWP_ASAN_HOOKS846 if (GuardedAlloc.pointerIsMine(Ptr))847 return true;848#endif // GWP_ASAN_HOOKS849 if (!Ptr || !isAligned(reinterpret_cast<uptr>(Ptr), MinAlignment))850 return false;851 Ptr = getHeaderTaggedPointer(const_cast<void *>(Ptr));852 Chunk::UnpackedHeader Header;853 return Chunk::isValid(Cookie, Ptr, &Header) &&854 Header.State == Chunk::State::Allocated;855 }856 857 bool useMemoryTaggingTestOnly() const {858 return useMemoryTagging<AllocatorConfig>(Primary.Options.load());859 }860 void disableMemoryTagging() {861 // If we haven't been initialized yet, we need to initialize now in order to862 // prevent a future call to initThreadMaybe() from enabling memory tagging863 // based on feature detection. But don't call initThreadMaybe() because it864 // may end up calling the allocator (via pthread_atfork, via the post-init865 // callback), which may cause mappings to be created with memory tagging866 // enabled.867 TSDRegistry.initOnceMaybe(this);868 if (allocatorSupportsMemoryTagging<AllocatorConfig>()) {869 Secondary.disableMemoryTagging();870 Primary.Options.clear(OptionBit::UseMemoryTagging);871 }872 }873 874 void setTrackAllocationStacks(bool Track) {875 initThreadMaybe();876 if (getFlags()->allocation_ring_buffer_size <= 0) {877 DCHECK(!Primary.Options.load().get(OptionBit::TrackAllocationStacks));878 return;879 }880 881 if (Track) {882 initRingBufferMaybe();883 Primary.Options.set(OptionBit::TrackAllocationStacks);884 } else885 Primary.Options.clear(OptionBit::TrackAllocationStacks);886 }887 888 void setFillContents(FillContentsMode FillContents) {889 initThreadMaybe();890 Primary.Options.setFillContentsMode(FillContents);891 }892 893 void setAddLargeAllocationSlack(bool AddSlack) {894 initThreadMaybe();895 if (AddSlack)896 Primary.Options.set(OptionBit::AddLargeAllocationSlack);897 else898 Primary.Options.clear(OptionBit::AddLargeAllocationSlack);899 }900 901 const char *getStackDepotAddress() {902 initThreadMaybe();903 AllocationRingBuffer *RB = getRingBuffer();904 return RB ? reinterpret_cast<char *>(RB->Depot) : nullptr;905 }906 907 uptr getStackDepotSize() {908 initThreadMaybe();909 AllocationRingBuffer *RB = getRingBuffer();910 return RB ? RB->StackDepotSize : 0;911 }912 913 const char *getRegionInfoArrayAddress() const {914 return Primary.getRegionInfoArrayAddress();915 }916 917 static uptr getRegionInfoArraySize() {918 return PrimaryT::getRegionInfoArraySize();919 }920 921 const char *getRingBufferAddress() {922 initThreadMaybe();923 return reinterpret_cast<char *>(getRingBuffer());924 }925 926 uptr getRingBufferSize() {927 initThreadMaybe();928 AllocationRingBuffer *RB = getRingBuffer();929 return RB && RB->RingBufferElements930 ? ringBufferSizeInBytes(RB->RingBufferElements)931 : 0;932 }933 934 static const uptr MaxTraceSize = 64;935 936 static void collectTraceMaybe(const StackDepot *Depot,937 uintptr_t (&Trace)[MaxTraceSize], u32 Hash) {938 uptr RingPos, Size;939 if (!Depot->find(Hash, &RingPos, &Size))940 return;941 for (unsigned I = 0; I != Size && I != MaxTraceSize; ++I)942 Trace[I] = static_cast<uintptr_t>(Depot->at(RingPos + I));943 }944 945 static void getErrorInfo(struct scudo_error_info *ErrorInfo,946 uintptr_t FaultAddr, const char *DepotPtr,947 size_t DepotSize, const char *RegionInfoPtr,948 const char *RingBufferPtr, size_t RingBufferSize,949 const char *Memory, const char *MemoryTags,950 uintptr_t MemoryAddr, size_t MemorySize) {951 // N.B. we need to support corrupted data in any of the buffers here. We get952 // this information from an external process (the crashing process) that953 // should not be able to crash the crash dumper (crash_dump on Android).954 // See also the get_error_info_fuzzer.955 *ErrorInfo = {};956 if (!allocatorSupportsMemoryTagging<AllocatorConfig>() ||957 MemoryAddr + MemorySize < MemoryAddr)958 return;959 960 const StackDepot *Depot = nullptr;961 if (DepotPtr) {962 // check for corrupted StackDepot. First we need to check whether we can963 // read the metadata, then whether the metadata matches the size.964 if (DepotSize < sizeof(*Depot))965 return;966 Depot = reinterpret_cast<const StackDepot *>(DepotPtr);967 if (!Depot->isValid(DepotSize))968 return;969 }970 971 size_t NextErrorReport = 0;972 973 // Check for OOB in the current block and the two surrounding blocks. Beyond974 // that, UAF is more likely.975 if (extractTag(FaultAddr) != 0)976 getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,977 RegionInfoPtr, Memory, MemoryTags, MemoryAddr,978 MemorySize, 0, 2);979 980 // Check the ring buffer. For primary allocations this will only find UAF;981 // for secondary allocations we can find either UAF or OOB.982 getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,983 RingBufferPtr, RingBufferSize);984 985 // Check for OOB in the 28 blocks surrounding the 3 we checked earlier.986 // Beyond that we are likely to hit false positives.987 if (extractTag(FaultAddr) != 0)988 getInlineErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot,989 RegionInfoPtr, Memory, MemoryTags, MemoryAddr,990 MemorySize, 2, 16);991 }992 993 uptr getBlockBeginTestOnly(const void *Ptr) {994 Chunk::UnpackedHeader Header;995 Chunk::loadHeader(Cookie, Ptr, &Header);996 DCHECK(Header.State == Chunk::State::Allocated);997 998 if (allocatorSupportsMemoryTagging<AllocatorConfig>())999 Ptr = untagPointer(const_cast<void *>(Ptr));1000 void *Begin = getBlockBegin(Ptr, &Header);1001 if (allocatorSupportsMemoryTagging<AllocatorConfig>())1002 Begin = untagPointer(Begin);1003 return reinterpret_cast<uptr>(Begin);1004 }1005 1006private:1007 typedef typename PrimaryT::SizeClassMap SizeClassMap;1008 1009 static const uptr MinAlignmentLog = SCUDO_MIN_ALIGNMENT_LOG;1010 static const uptr MaxAlignmentLog = 24U; // 16 MB seems reasonable.1011 static const uptr MinAlignment = 1UL << MinAlignmentLog;1012 static const uptr MaxAlignment = 1UL << MaxAlignmentLog;1013 static const uptr MaxAllowedMallocSize =1014 FIRST_32_SECOND_64(1UL << 31, 1ULL << 40);1015 1016 static_assert(MinAlignment >= sizeof(Chunk::PackedHeader),1017 "Minimal alignment must at least cover a chunk header.");1018 static_assert(!allocatorSupportsMemoryTagging<AllocatorConfig>() ||1019 MinAlignment >= archMemoryTagGranuleSize(),1020 "");1021 1022 static const u32 BlockMarker = 0x44554353U;1023 1024 // These are indexes into an "array" of 32-bit values that store information1025 // inline with a chunk that is relevant to diagnosing memory tag faults, where1026 // 0 corresponds to the address of the user memory. This means that only1027 // negative indexes may be used. The smallest index that may be used is -2,1028 // which corresponds to 8 bytes before the user memory, because the chunk1029 // header size is 8 bytes and in allocators that support memory tagging the1030 // minimum alignment is at least the tag granule size (16 on aarch64).1031 static const sptr MemTagAllocationTraceIndex = -2;1032 static const sptr MemTagAllocationTidIndex = -1;1033 1034 u32 Cookie = 0;1035 u32 QuarantineMaxChunkSize = 0;1036 1037 GlobalStats Stats;1038 PrimaryT Primary;1039 SecondaryT Secondary;1040 QuarantineT Quarantine;1041 TSDRegistryT TSDRegistry;1042 pthread_once_t PostInitNonce = PTHREAD_ONCE_INIT;1043 1044#ifdef GWP_ASAN_HOOKS1045 gwp_asan::GuardedPoolAllocator GuardedAlloc;1046 uptr GuardedAllocSlotSize = 0;1047#endif // GWP_ASAN_HOOKS1048 1049 struct AllocationRingBuffer {1050 struct Entry {1051 atomic_uptr Ptr;1052 atomic_uptr AllocationSize;1053 atomic_u32 AllocationTrace;1054 atomic_u32 AllocationTid;1055 atomic_u32 DeallocationTrace;1056 atomic_u32 DeallocationTid;1057 };1058 StackDepot *Depot = nullptr;1059 uptr StackDepotSize = 0;1060 MemMapT RawRingBufferMap;1061 MemMapT RawStackDepotMap;1062 u32 RingBufferElements = 0;1063 atomic_uptr Pos;1064 // An array of Size (at least one) elements of type Entry is immediately1065 // following to this struct.1066 };1067 static_assert(sizeof(AllocationRingBuffer) %1068 alignof(typename AllocationRingBuffer::Entry) ==1069 0,1070 "invalid alignment");1071 1072 // Lock to initialize the RingBuffer1073 HybridMutex RingBufferInitLock;1074 1075 // Pointer to memory mapped area starting with AllocationRingBuffer struct,1076 // and immediately followed by Size elements of type Entry.1077 atomic_uptr RingBufferAddress = {};1078 1079 AllocationRingBuffer *getRingBuffer() {1080 return reinterpret_cast<AllocationRingBuffer *>(1081 atomic_load(&RingBufferAddress, memory_order_acquire));1082 }1083 1084 // The following might get optimized out by the compiler.1085 NOINLINE void performSanityChecks() {1086 // Verify that the header offset field can hold the maximum offset. In the1087 // case of the Secondary allocator, it takes care of alignment and the1088 // offset will always be small. In the case of the Primary, the worst case1089 // scenario happens in the last size class, when the backend allocation1090 // would already be aligned on the requested alignment, which would happen1091 // to be the maximum alignment that would fit in that size class. As a1092 // result, the maximum offset will be at most the maximum alignment for the1093 // last size class minus the header size, in multiples of MinAlignment.1094 Chunk::UnpackedHeader Header = {};1095 const uptr MaxPrimaryAlignment = 1UL << getMostSignificantSetBitIndex(1096 SizeClassMap::MaxSize - MinAlignment);1097 const uptr MaxOffset =1098 (MaxPrimaryAlignment - Chunk::getHeaderSize()) >> MinAlignmentLog;1099 Header.Offset = MaxOffset & Chunk::OffsetMask;1100 if (UNLIKELY(Header.Offset != MaxOffset))1101 reportSanityCheckError("offset");1102 1103 // Verify that we can fit the maximum size or amount of unused bytes in the1104 // header. Given that the Secondary fits the allocation to a page, the worst1105 // case scenario happens in the Primary. It will depend on the second to1106 // last and last class sizes, as well as the dynamic base for the Primary.1107 // The following is an over-approximation that works for our needs.1108 const uptr MaxSizeOrUnusedBytes = SizeClassMap::MaxSize - 1;1109 Header.SizeOrUnusedBytes = MaxSizeOrUnusedBytes;1110 if (UNLIKELY(Header.SizeOrUnusedBytes != MaxSizeOrUnusedBytes))1111 reportSanityCheckError("size (or unused bytes)");1112 1113 const uptr LargestClassId = SizeClassMap::LargestClassId;1114 Header.ClassId = LargestClassId;1115 if (UNLIKELY(Header.ClassId != LargestClassId))1116 reportSanityCheckError("class ID");1117 }1118 1119 static inline void *getBlockBegin(const void *Ptr,1120 Chunk::UnpackedHeader *Header) {1121 return reinterpret_cast<void *>(1122 reinterpret_cast<uptr>(Ptr) - Chunk::getHeaderSize() -1123 (static_cast<uptr>(Header->Offset) << MinAlignmentLog));1124 }1125 1126 // Return the size of a chunk as requested during its allocation.1127 inline uptr getSize(const void *Ptr, Chunk::UnpackedHeader *Header) {1128 const uptr SizeOrUnusedBytes = Header->SizeOrUnusedBytes;1129 if (LIKELY(Header->ClassId))1130 return SizeOrUnusedBytes;1131 if (allocatorSupportsMemoryTagging<AllocatorConfig>())1132 Ptr = untagPointer(const_cast<void *>(Ptr));1133 return SecondaryT::getBlockEnd(getBlockBegin(Ptr, Header)) -1134 reinterpret_cast<uptr>(Ptr) - SizeOrUnusedBytes;1135 }1136 1137 ALWAYS_INLINE void *initChunk(const uptr ClassId, const Chunk::Origin Origin,1138 void *Block, const uptr UserPtr,1139 const uptr SizeOrUnusedBytes,1140 const FillContentsMode FillContents) {1141 // Compute the default pointer before adding the header tag1142 const uptr DefaultAlignedPtr =1143 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();1144 1145 Block = addHeaderTag(Block);1146 // Only do content fill when it's from primary allocator because secondary1147 // allocator has filled the content.1148 if (ClassId != 0 && UNLIKELY(FillContents != NoFill)) {1149 // This condition is not necessarily unlikely, but since memset is1150 // costly, we might as well mark it as such.1151 memset(Block, FillContents == ZeroFill ? 0 : PatternFillByte,1152 PrimaryT::getSizeByClassId(ClassId));1153 }1154 1155 Chunk::UnpackedHeader Header = {};1156 1157 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {1158 const uptr Offset = UserPtr - DefaultAlignedPtr;1159 DCHECK_GE(Offset, 2 * sizeof(u32));1160 // The BlockMarker has no security purpose, but is specifically meant for1161 // the chunk iteration function that can be used in debugging situations.1162 // It is the only situation where we have to locate the start of a chunk1163 // based on its block address.1164 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;1165 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);1166 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;1167 }1168 1169 Header.ClassId = ClassId & Chunk::ClassIdMask;1170 Header.State = Chunk::State::Allocated;1171 Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;1172 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;1173 Chunk::storeHeader(Cookie, reinterpret_cast<void *>(addHeaderTag(UserPtr)),1174 &Header);1175 1176 return reinterpret_cast<void *>(UserPtr);1177 }1178 1179 NOINLINE void *1180 initChunkWithMemoryTagging(const uptr ClassId, const Chunk::Origin Origin,1181 void *Block, const uptr UserPtr, const uptr Size,1182 const uptr SizeOrUnusedBytes,1183 const FillContentsMode FillContents) {1184 const Options Options = Primary.Options.load();1185 DCHECK(useMemoryTagging<AllocatorConfig>(Options));1186 1187 // Compute the default pointer before adding the header tag1188 const uptr DefaultAlignedPtr =1189 reinterpret_cast<uptr>(Block) + Chunk::getHeaderSize();1190 1191 void *Ptr = reinterpret_cast<void *>(UserPtr);1192 void *TaggedPtr = Ptr;1193 1194 if (LIKELY(ClassId)) {1195 // Init the primary chunk.1196 //1197 // We only need to zero or tag the contents for Primary backed1198 // allocations. We only set tags for primary allocations in order to avoid1199 // faulting potentially large numbers of pages for large secondary1200 // allocations. We assume that guard pages are enough to protect these1201 // allocations.1202 //1203 // FIXME: When the kernel provides a way to set the background tag of a1204 // mapping, we should be able to tag secondary allocations as well.1205 //1206 // When memory tagging is enabled, zeroing the contents is done as part of1207 // setting the tag.1208 1209 Chunk::UnpackedHeader Header;1210 const uptr BlockSize = PrimaryT::getSizeByClassId(ClassId);1211 const uptr BlockUptr = reinterpret_cast<uptr>(Block);1212 const uptr BlockEnd = BlockUptr + BlockSize;1213 // If possible, try to reuse the UAF tag that was set by deallocate().1214 // For simplicity, only reuse tags if we have the same start address as1215 // the previous allocation. This handles the majority of cases since1216 // most allocations will not be more aligned than the minimum alignment.1217 //1218 // We need to handle situations involving reclaimed chunks, and retag1219 // the reclaimed portions if necessary. In the case where the chunk is1220 // fully reclaimed, the chunk's header will be zero, which will trigger1221 // the code path for new mappings and invalid chunks that prepares the1222 // chunk from scratch. There are three possibilities for partial1223 // reclaiming:1224 //1225 // (1) Header was reclaimed, data was partially reclaimed.1226 // (2) Header was not reclaimed, all data was reclaimed (e.g. because1227 // data started on a page boundary).1228 // (3) Header was not reclaimed, data was partially reclaimed.1229 //1230 // Case (1) will be handled in the same way as for full reclaiming,1231 // since the header will be zero.1232 //1233 // We can detect case (2) by loading the tag from the start1234 // of the chunk. If it is zero, it means that either all data was1235 // reclaimed (since we never use zero as the chunk tag), or that the1236 // previous allocation was of size zero. Either way, we need to prepare1237 // a new chunk from scratch.1238 //1239 // We can detect case (3) by moving to the next page (if covered by the1240 // chunk) and loading the tag of its first granule. If it is zero, it1241 // means that all following pages may need to be retagged. On the other1242 // hand, if it is nonzero, we can assume that all following pages are1243 // still tagged, according to the logic that if any of the pages1244 // following the next page were reclaimed, the next page would have been1245 // reclaimed as well.1246 uptr TaggedUserPtr;1247 uptr PrevUserPtr;1248 if (getChunkFromBlock(BlockUptr, &PrevUserPtr, &Header) &&1249 PrevUserPtr == UserPtr &&1250 (TaggedUserPtr = loadTag(UserPtr)) != UserPtr) {1251 uptr PrevEnd = TaggedUserPtr + Header.SizeOrUnusedBytes;1252 const uptr NextPage = roundUp(TaggedUserPtr, getPageSizeCached());1253 if (NextPage < PrevEnd && loadTag(NextPage) != NextPage)1254 PrevEnd = NextPage;1255 TaggedPtr = reinterpret_cast<void *>(TaggedUserPtr);1256 resizeTaggedChunk(PrevEnd, TaggedUserPtr + Size, Size, BlockEnd);1257 if (UNLIKELY(FillContents != NoFill && !Header.OriginOrWasZeroed)) {1258 // If an allocation needs to be zeroed (i.e. calloc) we can normally1259 // avoid zeroing the memory now since we can rely on memory having1260 // been zeroed on free, as this is normally done while setting the1261 // UAF tag. But if tagging was disabled per-thread when the memory1262 // was freed, it would not have been retagged and thus zeroed, and1263 // therefore it needs to be zeroed now.1264 memset(TaggedPtr, 0,1265 Min(Size, roundUp(PrevEnd - TaggedUserPtr,1266 archMemoryTagGranuleSize())));1267 } else if (Size) {1268 // Clear any stack metadata that may have previously been stored in1269 // the chunk data.1270 memset(TaggedPtr, 0, archMemoryTagGranuleSize());1271 }1272 } else {1273 const uptr OddEvenMask =1274 computeOddEvenMaskForPointerMaybe(Options, BlockUptr, ClassId);1275 TaggedPtr = prepareTaggedChunk(Ptr, Size, OddEvenMask, BlockEnd);1276 }1277 storePrimaryAllocationStackMaybe(Options, Ptr);1278 } else {1279 // Init the secondary chunk.1280 1281 Block = addHeaderTag(Block);1282 Ptr = addHeaderTag(Ptr);1283 storeTags(reinterpret_cast<uptr>(Block), reinterpret_cast<uptr>(Ptr));1284 storeSecondaryAllocationStackMaybe(Options, Ptr, Size);1285 }1286 1287 Chunk::UnpackedHeader Header = {};1288 1289 if (UNLIKELY(DefaultAlignedPtr != UserPtr)) {1290 const uptr Offset = UserPtr - DefaultAlignedPtr;1291 DCHECK_GE(Offset, 2 * sizeof(u32));1292 // The BlockMarker has no security purpose, but is specifically meant for1293 // the chunk iteration function that can be used in debugging situations.1294 // It is the only situation where we have to locate the start of a chunk1295 // based on its block address.1296 reinterpret_cast<u32 *>(Block)[0] = BlockMarker;1297 reinterpret_cast<u32 *>(Block)[1] = static_cast<u32>(Offset);1298 Header.Offset = (Offset >> MinAlignmentLog) & Chunk::OffsetMask;1299 }1300 1301 Header.ClassId = ClassId & Chunk::ClassIdMask;1302 Header.State = Chunk::State::Allocated;1303 Header.OriginOrWasZeroed = Origin & Chunk::OriginMask;1304 Header.SizeOrUnusedBytes = SizeOrUnusedBytes & Chunk::SizeOrUnusedBytesMask;1305 Chunk::storeHeader(Cookie, Ptr, &Header);1306 1307 return TaggedPtr;1308 }1309 1310 void quarantineOrDeallocateChunk(const Options &Options, void *TaggedPtr,1311 Chunk::UnpackedHeader *Header,1312 uptr Size) NO_THREAD_SAFETY_ANALYSIS {1313 void *Ptr = getHeaderTaggedPointer(TaggedPtr);1314 // If the quarantine is disabled, the actual size of a chunk is 0 or larger1315 // than the maximum allowed, we return a chunk directly to the backend.1316 // This purposefully underflows for Size == 0.1317 const bool BypassQuarantine = AllocatorConfig::getQuarantineDisabled() ||1318 !Quarantine.getCacheSize() ||1319 ((Size - 1) >= QuarantineMaxChunkSize) ||1320 !Header->ClassId;1321 if (BypassQuarantine)1322 Header->State = Chunk::State::Available;1323 else1324 Header->State = Chunk::State::Quarantined;1325 1326 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options)))1327 Header->OriginOrWasZeroed = 0U;1328 else {1329 Header->OriginOrWasZeroed =1330 Header->ClassId && !TSDRegistry.getDisableMemInit();1331 }1332 1333 Chunk::storeHeader(Cookie, Ptr, Header);1334 1335 if (BypassQuarantine) {1336 void *BlockBegin;1337 if (LIKELY(!useMemoryTagging<AllocatorConfig>(Options))) {1338 // Must do this after storeHeader because loadHeader uses a tagged ptr.1339 if (allocatorSupportsMemoryTagging<AllocatorConfig>())1340 Ptr = untagPointer(Ptr);1341 BlockBegin = getBlockBegin(Ptr, Header);1342 } else {1343 BlockBegin = retagBlock(Options, TaggedPtr, Ptr, Header, Size, true);1344 }1345 1346 const uptr ClassId = Header->ClassId;1347 if (LIKELY(ClassId)) {1348 bool CacheDrained;1349 {1350 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);1351 CacheDrained =1352 TSD->getSizeClassAllocator().deallocate(ClassId, BlockBegin);1353 }1354 // When we have drained some blocks back to the Primary from TSD, that1355 // implies that we may have the chance to release some pages as well.1356 // Note that in order not to block other thread's accessing the TSD,1357 // release the TSD first then try the page release.1358 if (CacheDrained)1359 Primary.tryReleaseToOS(ClassId, ReleaseToOS::Normal);1360 } else {1361 Secondary.deallocate(Options, BlockBegin);1362 }1363 } else {1364 if (UNLIKELY(useMemoryTagging<AllocatorConfig>(Options)))1365 retagBlock(Options, TaggedPtr, Ptr, Header, Size, false);1366 typename TSDRegistryT::ScopedTSD TSD(TSDRegistry);1367 Quarantine.put(&TSD->getQuarantineCache(),1368 QuarantineCallback(*this, TSD->getSizeClassAllocator()),1369 Ptr, Size);1370 }1371 }1372 1373 NOINLINE void *retagBlock(const Options &Options, void *TaggedPtr, void *&Ptr,1374 Chunk::UnpackedHeader *Header, const uptr Size,1375 bool BypassQuarantine) {1376 DCHECK(useMemoryTagging<AllocatorConfig>(Options));1377 1378 const u8 PrevTag = extractTag(reinterpret_cast<uptr>(TaggedPtr));1379 storeDeallocationStackMaybe(Options, Ptr, PrevTag, Size);1380 if (Header->ClassId && !TSDRegistry.getDisableMemInit()) {1381 uptr TaggedBegin, TaggedEnd;1382 const uptr OddEvenMask = computeOddEvenMaskForPointerMaybe(1383 Options, reinterpret_cast<uptr>(getBlockBegin(Ptr, Header)),1384 Header->ClassId);1385 // Exclude the previous tag so that immediate use after free is1386 // detected 100% of the time.1387 setRandomTag(Ptr, Size, OddEvenMask | (1UL << PrevTag), &TaggedBegin,1388 &TaggedEnd);1389 }1390 1391 Ptr = untagPointer(Ptr);1392 void *BlockBegin = getBlockBegin(Ptr, Header);1393 if (BypassQuarantine && !Header->ClassId) {1394 storeTags(reinterpret_cast<uptr>(BlockBegin),1395 reinterpret_cast<uptr>(Ptr));1396 }1397 1398 return BlockBegin;1399 }1400 1401 bool getChunkFromBlock(uptr Block, uptr *Chunk,1402 Chunk::UnpackedHeader *Header) {1403 *Chunk =1404 Block + getChunkOffsetFromBlock(reinterpret_cast<const char *>(Block));1405 return Chunk::isValid(Cookie, reinterpret_cast<void *>(*Chunk), Header);1406 }1407 1408 static uptr getChunkOffsetFromBlock(const char *Block) {1409 u32 Offset = 0;1410 if (reinterpret_cast<const u32 *>(Block)[0] == BlockMarker)1411 Offset = reinterpret_cast<const u32 *>(Block)[1];1412 return Offset + Chunk::getHeaderSize();1413 }1414 1415 // Set the tag of the granule past the end of the allocation to 0, to catch1416 // linear overflows even if a previous larger allocation used the same block1417 // and tag. Only do this if the granule past the end is in our block, because1418 // this would otherwise lead to a SEGV if the allocation covers the entire1419 // block and our block is at the end of a mapping. The tag of the next block's1420 // header granule will be set to 0, so it will serve the purpose of catching1421 // linear overflows in this case.1422 //1423 // For allocations of size 0 we do not end up storing the address tag to the1424 // memory tag space, which getInlineErrorInfo() normally relies on to match1425 // address tags against chunks. To allow matching in this case we store the1426 // address tag in the first byte of the chunk.1427 void storeEndMarker(uptr End, uptr Size, uptr BlockEnd) {1428 DCHECK_EQ(BlockEnd, untagPointer(BlockEnd));1429 uptr UntaggedEnd = untagPointer(End);1430 if (UntaggedEnd != BlockEnd) {1431 storeTag(UntaggedEnd);1432 if (Size == 0)1433 *reinterpret_cast<u8 *>(UntaggedEnd) = extractTag(End);1434 }1435 }1436 1437 void *prepareTaggedChunk(void *Ptr, uptr Size, uptr ExcludeMask,1438 uptr BlockEnd) {1439 // Prepare the granule before the chunk to store the chunk header by setting1440 // its tag to 0. Normally its tag will already be 0, but in the case where a1441 // chunk holding a low alignment allocation is reused for a higher alignment1442 // allocation, the chunk may already have a non-zero tag from the previous1443 // allocation.1444 storeTag(reinterpret_cast<uptr>(Ptr) - archMemoryTagGranuleSize());1445 1446 uptr TaggedBegin, TaggedEnd;1447 setRandomTag(Ptr, Size, ExcludeMask, &TaggedBegin, &TaggedEnd);1448 1449 storeEndMarker(TaggedEnd, Size, BlockEnd);1450 return reinterpret_cast<void *>(TaggedBegin);1451 }1452 1453 void resizeTaggedChunk(uptr OldPtr, uptr NewPtr, uptr NewSize,1454 uptr BlockEnd) {1455 uptr RoundOldPtr = roundUp(OldPtr, archMemoryTagGranuleSize());1456 uptr RoundNewPtr;1457 if (RoundOldPtr >= NewPtr) {1458 // If the allocation is shrinking we just need to set the tag past the end1459 // of the allocation to 0. See explanation in storeEndMarker() above.1460 RoundNewPtr = roundUp(NewPtr, archMemoryTagGranuleSize());1461 } else {1462 // Set the memory tag of the region1463 // [RoundOldPtr, roundUp(NewPtr, archMemoryTagGranuleSize()))1464 // to the pointer tag stored in OldPtr.1465 RoundNewPtr = storeTags(RoundOldPtr, NewPtr);1466 }1467 storeEndMarker(RoundNewPtr, NewSize, BlockEnd);1468 }1469 1470 void storePrimaryAllocationStackMaybe(const Options &Options, void *Ptr) {1471 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))1472 return;1473 AllocationRingBuffer *RB = getRingBuffer();1474 if (!RB)1475 return;1476 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);1477 Ptr32[MemTagAllocationTraceIndex] = collectStackTrace(RB->Depot);1478 Ptr32[MemTagAllocationTidIndex] = getThreadID();1479 }1480 1481 void storeRingBufferEntry(AllocationRingBuffer *RB, void *Ptr,1482 u32 AllocationTrace, u32 AllocationTid,1483 uptr AllocationSize, u32 DeallocationTrace,1484 u32 DeallocationTid) {1485 uptr Pos = atomic_fetch_add(&RB->Pos, 1, memory_order_relaxed);1486 typename AllocationRingBuffer::Entry *Entry =1487 getRingBufferEntry(RB, Pos % RB->RingBufferElements);1488 1489 // First invalidate our entry so that we don't attempt to interpret a1490 // partially written state in getSecondaryErrorInfo(). The fences below1491 // ensure that the compiler does not move the stores to Ptr in between the1492 // stores to the other fields.1493 atomic_store_relaxed(&Entry->Ptr, 0);1494 1495 __atomic_signal_fence(__ATOMIC_SEQ_CST);1496 atomic_store_relaxed(&Entry->AllocationTrace, AllocationTrace);1497 atomic_store_relaxed(&Entry->AllocationTid, AllocationTid);1498 atomic_store_relaxed(&Entry->AllocationSize, AllocationSize);1499 atomic_store_relaxed(&Entry->DeallocationTrace, DeallocationTrace);1500 atomic_store_relaxed(&Entry->DeallocationTid, DeallocationTid);1501 __atomic_signal_fence(__ATOMIC_SEQ_CST);1502 1503 atomic_store_relaxed(&Entry->Ptr, reinterpret_cast<uptr>(Ptr));1504 }1505 1506 void storeSecondaryAllocationStackMaybe(const Options &Options, void *Ptr,1507 uptr Size) {1508 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))1509 return;1510 AllocationRingBuffer *RB = getRingBuffer();1511 if (!RB)1512 return;1513 u32 Trace = collectStackTrace(RB->Depot);1514 u32 Tid = getThreadID();1515 1516 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);1517 Ptr32[MemTagAllocationTraceIndex] = Trace;1518 Ptr32[MemTagAllocationTidIndex] = Tid;1519 1520 storeRingBufferEntry(RB, untagPointer(Ptr), Trace, Tid, Size, 0, 0);1521 }1522 1523 void storeDeallocationStackMaybe(const Options &Options, void *Ptr,1524 u8 PrevTag, uptr Size) {1525 if (!UNLIKELY(Options.get(OptionBit::TrackAllocationStacks)))1526 return;1527 AllocationRingBuffer *RB = getRingBuffer();1528 if (!RB)1529 return;1530 auto *Ptr32 = reinterpret_cast<u32 *>(Ptr);1531 u32 AllocationTrace = Ptr32[MemTagAllocationTraceIndex];1532 u32 AllocationTid = Ptr32[MemTagAllocationTidIndex];1533 1534 u32 DeallocationTrace = collectStackTrace(RB->Depot);1535 u32 DeallocationTid = getThreadID();1536 1537 storeRingBufferEntry(RB, addFixedTag(untagPointer(Ptr), PrevTag),1538 AllocationTrace, AllocationTid, Size,1539 DeallocationTrace, DeallocationTid);1540 }1541 1542 static const size_t NumErrorReports =1543 sizeof(((scudo_error_info *)nullptr)->reports) /1544 sizeof(((scudo_error_info *)nullptr)->reports[0]);1545 1546 static void getInlineErrorInfo(struct scudo_error_info *ErrorInfo,1547 size_t &NextErrorReport, uintptr_t FaultAddr,1548 const StackDepot *Depot,1549 const char *RegionInfoPtr, const char *Memory,1550 const char *MemoryTags, uintptr_t MemoryAddr,1551 size_t MemorySize, size_t MinDistance,1552 size_t MaxDistance) {1553 uptr UntaggedFaultAddr = untagPointer(FaultAddr);1554 u8 FaultAddrTag = extractTag(FaultAddr);1555 BlockInfo Info =1556 PrimaryT::findNearestBlock(RegionInfoPtr, UntaggedFaultAddr);1557 1558 auto GetGranule = [&](uptr Addr, const char **Data, uint8_t *Tag) -> bool {1559 if (Addr < MemoryAddr || Addr + archMemoryTagGranuleSize() < Addr ||1560 Addr + archMemoryTagGranuleSize() > MemoryAddr + MemorySize)1561 return false;1562 *Data = &Memory[Addr - MemoryAddr];1563 *Tag = static_cast<u8>(1564 MemoryTags[(Addr - MemoryAddr) / archMemoryTagGranuleSize()]);1565 return true;1566 };1567 1568 auto ReadBlock = [&](uptr Addr, uptr *ChunkAddr,1569 Chunk::UnpackedHeader *Header, const u32 **Data,1570 u8 *Tag) {1571 const char *BlockBegin;1572 u8 BlockBeginTag;1573 if (!GetGranule(Addr, &BlockBegin, &BlockBeginTag))1574 return false;1575 uptr ChunkOffset = getChunkOffsetFromBlock(BlockBegin);1576 *ChunkAddr = Addr + ChunkOffset;1577 1578 const char *ChunkBegin;1579 if (!GetGranule(*ChunkAddr, &ChunkBegin, Tag))1580 return false;1581 *Header = *reinterpret_cast<const Chunk::UnpackedHeader *>(1582 ChunkBegin - Chunk::getHeaderSize());1583 *Data = reinterpret_cast<const u32 *>(ChunkBegin);1584 1585 // Allocations of size 0 will have stashed the tag in the first byte of1586 // the chunk, see storeEndMarker().1587 if (Header->SizeOrUnusedBytes == 0)1588 *Tag = static_cast<u8>(*ChunkBegin);1589 1590 return true;1591 };1592 1593 if (NextErrorReport == NumErrorReports)1594 return;1595 1596 auto CheckOOB = [&](uptr BlockAddr) {1597 if (BlockAddr < Info.RegionBegin || BlockAddr >= Info.RegionEnd)1598 return false;1599 1600 uptr ChunkAddr;1601 Chunk::UnpackedHeader Header;1602 const u32 *Data;1603 uint8_t Tag;1604 if (!ReadBlock(BlockAddr, &ChunkAddr, &Header, &Data, &Tag) ||1605 Header.State != Chunk::State::Allocated || Tag != FaultAddrTag)1606 return false;1607 1608 auto *R = &ErrorInfo->reports[NextErrorReport++];1609 R->error_type =1610 UntaggedFaultAddr < ChunkAddr ? BUFFER_UNDERFLOW : BUFFER_OVERFLOW;1611 R->allocation_address = ChunkAddr;1612 R->allocation_size = Header.SizeOrUnusedBytes;1613 if (Depot) {1614 collectTraceMaybe(Depot, R->allocation_trace,1615 Data[MemTagAllocationTraceIndex]);1616 }1617 R->allocation_tid = Data[MemTagAllocationTidIndex];1618 return NextErrorReport == NumErrorReports;1619 };1620 1621 if (MinDistance == 0 && CheckOOB(Info.BlockBegin))1622 return;1623 1624 for (size_t I = Max<size_t>(MinDistance, 1); I != MaxDistance; ++I)1625 if (CheckOOB(Info.BlockBegin + I * Info.BlockSize) ||1626 CheckOOB(Info.BlockBegin - I * Info.BlockSize))1627 return;1628 }1629 1630 static void getRingBufferErrorInfo(struct scudo_error_info *ErrorInfo,1631 size_t &NextErrorReport,1632 uintptr_t FaultAddr,1633 const StackDepot *Depot,1634 const char *RingBufferPtr,1635 size_t RingBufferSize) {1636 auto *RingBuffer =1637 reinterpret_cast<const AllocationRingBuffer *>(RingBufferPtr);1638 size_t RingBufferElements = ringBufferElementsFromBytes(RingBufferSize);1639 if (!RingBuffer || RingBufferElements == 0 || !Depot)1640 return;1641 uptr Pos = atomic_load_relaxed(&RingBuffer->Pos);1642 1643 for (uptr I = Pos - 1; I != Pos - 1 - RingBufferElements &&1644 NextErrorReport != NumErrorReports;1645 --I) {1646 auto *Entry = getRingBufferEntry(RingBuffer, I % RingBufferElements);1647 uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr);1648 if (!EntryPtr)1649 continue;1650 1651 uptr UntaggedEntryPtr = untagPointer(EntryPtr);1652 uptr EntrySize = atomic_load_relaxed(&Entry->AllocationSize);1653 u32 AllocationTrace = atomic_load_relaxed(&Entry->AllocationTrace);1654 u32 AllocationTid = atomic_load_relaxed(&Entry->AllocationTid);1655 u32 DeallocationTrace = atomic_load_relaxed(&Entry->DeallocationTrace);1656 u32 DeallocationTid = atomic_load_relaxed(&Entry->DeallocationTid);1657 1658 if (DeallocationTid) {1659 // For UAF we only consider in-bounds fault addresses because1660 // out-of-bounds UAF is rare and attempting to detect it is very likely1661 // to result in false positives.1662 if (FaultAddr < EntryPtr || FaultAddr >= EntryPtr + EntrySize)1663 continue;1664 } else {1665 // Ring buffer OOB is only possible with secondary allocations. In this1666 // case we are guaranteed a guard region of at least a page on either1667 // side of the allocation (guard page on the right, guard page + tagged1668 // region on the left), so ignore any faults outside of that range.1669 if (FaultAddr < EntryPtr - getPageSizeCached() ||1670 FaultAddr >= EntryPtr + EntrySize + getPageSizeCached())1671 continue;1672 1673 // For UAF the ring buffer will contain two entries, one for the1674 // allocation and another for the deallocation. Don't report buffer1675 // overflow/underflow using the allocation entry if we have already1676 // collected a report from the deallocation entry.1677 bool Found = false;1678 for (uptr J = 0; J != NextErrorReport; ++J) {1679 if (ErrorInfo->reports[J].allocation_address == UntaggedEntryPtr) {1680 Found = true;1681 break;1682 }1683 }1684 if (Found)1685 continue;1686 }1687 1688 auto *R = &ErrorInfo->reports[NextErrorReport++];1689 if (DeallocationTid)1690 R->error_type = USE_AFTER_FREE;1691 else if (FaultAddr < EntryPtr)1692 R->error_type = BUFFER_UNDERFLOW;1693 else1694 R->error_type = BUFFER_OVERFLOW;1695 1696 R->allocation_address = UntaggedEntryPtr;1697 R->allocation_size = EntrySize;1698 collectTraceMaybe(Depot, R->allocation_trace, AllocationTrace);1699 R->allocation_tid = AllocationTid;1700 collectTraceMaybe(Depot, R->deallocation_trace, DeallocationTrace);1701 R->deallocation_tid = DeallocationTid;1702 }1703 }1704 1705 uptr getStats(ScopedString *Str) {1706 Primary.getStats(Str);1707 Secondary.getStats(Str);1708 if (!AllocatorConfig::getQuarantineDisabled())1709 Quarantine.getStats(Str);1710 TSDRegistry.getStats(Str);1711 return Str->length();1712 }1713 1714 static typename AllocationRingBuffer::Entry *1715 getRingBufferEntry(AllocationRingBuffer *RB, uptr N) {1716 char *RBEntryStart =1717 &reinterpret_cast<char *>(RB)[sizeof(AllocationRingBuffer)];1718 return &reinterpret_cast<typename AllocationRingBuffer::Entry *>(1719 RBEntryStart)[N];1720 }1721 static const typename AllocationRingBuffer::Entry *1722 getRingBufferEntry(const AllocationRingBuffer *RB, uptr N) {1723 const char *RBEntryStart =1724 &reinterpret_cast<const char *>(RB)[sizeof(AllocationRingBuffer)];1725 return &reinterpret_cast<const typename AllocationRingBuffer::Entry *>(1726 RBEntryStart)[N];1727 }1728 1729 void initRingBufferMaybe() {1730 ScopedLock L(RingBufferInitLock);1731 if (getRingBuffer() != nullptr)1732 return;1733 1734 int ring_buffer_size = getFlags()->allocation_ring_buffer_size;1735 if (ring_buffer_size <= 0)1736 return;1737 1738 u32 AllocationRingBufferSize = static_cast<u32>(ring_buffer_size);1739 1740 // We store alloc and free stacks for each entry.1741 constexpr u32 kStacksPerRingBufferEntry = 2;1742 constexpr u32 kMaxU32Pow2 = ~(UINT32_MAX >> 1);1743 static_assert(isPowerOfTwo(kMaxU32Pow2));1744 // On Android we always have 3 frames at the bottom: __start_main,1745 // __libc_init, main, and 3 at the top: malloc, scudo_malloc and1746 // Allocator::allocate. This leaves 10 frames for the user app. The next1747 // smallest power of two (8) would only leave 2, which is clearly too1748 // little.1749 constexpr u32 kFramesPerStack = 16;1750 static_assert(isPowerOfTwo(kFramesPerStack));1751 1752 if (AllocationRingBufferSize > kMaxU32Pow2 / kStacksPerRingBufferEntry)1753 return;1754 u32 TabSize = static_cast<u32>(roundUpPowerOfTwo(kStacksPerRingBufferEntry *1755 AllocationRingBufferSize));1756 if (TabSize > UINT32_MAX / kFramesPerStack)1757 return;1758 u32 RingSize = static_cast<u32>(TabSize * kFramesPerStack);1759 1760 uptr StackDepotSize = sizeof(StackDepot) + sizeof(atomic_u64) * RingSize +1761 sizeof(atomic_u32) * TabSize;1762 MemMapT DepotMap;1763 DepotMap.map(1764 /*Addr=*/0U, roundUp(StackDepotSize, getPageSizeCached()),1765 "scudo:stack_depot");1766 auto *Depot = reinterpret_cast<StackDepot *>(DepotMap.getBase());1767 Depot->init(RingSize, TabSize);1768 1769 MemMapT MemMap;1770 MemMap.map(1771 /*Addr=*/0U,1772 roundUp(ringBufferSizeInBytes(AllocationRingBufferSize),1773 getPageSizeCached()),1774 "scudo:ring_buffer");1775 auto *RB = reinterpret_cast<AllocationRingBuffer *>(MemMap.getBase());1776 RB->RawRingBufferMap = MemMap;1777 RB->RingBufferElements = AllocationRingBufferSize;1778 RB->Depot = Depot;1779 RB->StackDepotSize = StackDepotSize;1780 RB->RawStackDepotMap = DepotMap;1781 1782 atomic_store(&RingBufferAddress, reinterpret_cast<uptr>(RB),1783 memory_order_release);1784 }1785 1786 void unmapRingBuffer() {1787 AllocationRingBuffer *RB = getRingBuffer();1788 if (RB == nullptr)1789 return;1790 // N.B. because RawStackDepotMap is part of RawRingBufferMap, the order1791 // is very important.1792 RB->RawStackDepotMap.unmap();1793 // Note that the `RB->RawRingBufferMap` is stored on the pages managed by1794 // itself. Take over the ownership before calling unmap() so that any1795 // operation along with unmap() won't touch inaccessible pages.1796 MemMapT RawRingBufferMap = RB->RawRingBufferMap;1797 RawRingBufferMap.unmap();1798 atomic_store(&RingBufferAddress, 0, memory_order_release);1799 }1800 1801 static constexpr size_t ringBufferSizeInBytes(u32 RingBufferElements) {1802 return sizeof(AllocationRingBuffer) +1803 RingBufferElements * sizeof(typename AllocationRingBuffer::Entry);1804 }1805 1806 static constexpr size_t ringBufferElementsFromBytes(size_t Bytes) {1807 if (Bytes < sizeof(AllocationRingBuffer)) {1808 return 0;1809 }1810 return (Bytes - sizeof(AllocationRingBuffer)) /1811 sizeof(typename AllocationRingBuffer::Entry);1812 }1813};1814 1815} // namespace scudo1816 1817#endif // SCUDO_COMBINED_H_1818