272 lines · c
1//===-- tsd_shared.h --------------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#ifndef SCUDO_TSD_SHARED_H_10#define SCUDO_TSD_SHARED_H_11 12#include "tsd.h"13 14#include "string_utils.h"15 16#if SCUDO_HAS_PLATFORM_TLS_SLOT17// This is a platform-provided header that needs to be on the include path when18// Scudo is compiled. It must declare a function with the prototype:19// uintptr_t *getPlatformAllocatorTlsSlot()20// that returns the address of a thread-local word of storage reserved for21// Scudo, that must be zero-initialized in newly created threads.22#include "scudo_platform_tls_slot.h"23#endif24 25namespace scudo {26 27template <class Allocator, u32 TSDsArraySize, u32 DefaultTSDCount>28struct TSDRegistrySharedT {29 using ThisT = TSDRegistrySharedT<Allocator, TSDsArraySize, DefaultTSDCount>;30 31 struct ScopedTSD {32 ALWAYS_INLINE ScopedTSD(ThisT &TSDRegistry) {33 CurrentTSD = TSDRegistry.getTSDAndLock();34 DCHECK_NE(CurrentTSD, nullptr);35 }36 37 ~ScopedTSD() { CurrentTSD->unlock(); }38 39 TSD<Allocator> &operator*() { return *CurrentTSD; }40 41 TSD<Allocator> *operator->() {42 CurrentTSD->assertLocked(/*BypassCheck=*/false);43 return CurrentTSD;44 }45 46 private:47 TSD<Allocator> *CurrentTSD;48 };49 50 void init(Allocator *Instance) REQUIRES(Mutex) {51 DCHECK(!Initialized);52 Instance->init();53 for (u32 I = 0; I < TSDsArraySize; I++)54 TSDs[I].init(Instance);55 const u32 NumberOfCPUs = getNumberOfCPUs();56 setNumberOfTSDs((NumberOfCPUs == 0) ? DefaultTSDCount57 : Min(NumberOfCPUs, DefaultTSDCount));58 Initialized = true;59 }60 61 void initOnceMaybe(Allocator *Instance) EXCLUDES(Mutex) {62 ScopedLock L(Mutex);63 if (LIKELY(Initialized))64 return;65 init(Instance); // Sets Initialized.66 }67 68 void unmapTestOnly(Allocator *Instance) EXCLUDES(Mutex) {69 for (u32 I = 0; I < TSDsArraySize; I++) {70 TSDs[I].commitBack(Instance);71 TSDs[I] = {};72 }73 setCurrentTSD(nullptr);74 ScopedLock L(Mutex);75 Initialized = false;76 }77 78 void drainCaches(Allocator *Instance) {79 ScopedLock L(MutexTSDs);80 for (uptr I = 0; I < NumberOfTSDs; ++I) {81 TSDs[I].lock();82 Instance->drainCache(&TSDs[I]);83 TSDs[I].unlock();84 }85 }86 87 ALWAYS_INLINE void initThreadMaybe(Allocator *Instance,88 UNUSED bool MinimalInit) {89 if (LIKELY(getCurrentTSD()))90 return;91 initThread(Instance);92 }93 94 void disable() NO_THREAD_SAFETY_ANALYSIS {95 Mutex.lock();96 MutexTSDs.lock();97 for (u32 I = 0; I < TSDsArraySize; I++)98 TSDs[I].lock();99 }100 101 void enable() NO_THREAD_SAFETY_ANALYSIS {102 for (s32 I = static_cast<s32>(TSDsArraySize - 1); I >= 0; I--)103 TSDs[I].unlock();104 MutexTSDs.unlock();105 Mutex.unlock();106 }107 108 bool setOption(Option O, sptr Value) {109 if (O == Option::MaxTSDsCount)110 return setNumberOfTSDs(static_cast<u32>(Value));111 if (O == Option::ThreadDisableMemInit)112 setDisableMemInit(Value);113 // Not supported by the TSD Registry, but not an error either.114 return true;115 }116 117 bool getDisableMemInit() const { return *getTlsPtr() & 1; }118 119 void getStats(ScopedString *Str) EXCLUDES(MutexTSDs) {120 ScopedLock L(MutexTSDs);121 122 Str->append("Stats: SharedTSDs: %u available; total %u\n", NumberOfTSDs,123 TSDsArraySize);124 for (uptr I = 0; I < NumberOfTSDs; ++I) {125 TSDs[I].lock();126 // Theoretically, we want to mark TSD::lock()/TSD::unlock() with proper127 // thread annotations. However, given the TSD is only locked on shared128 // path, do the assertion in a separate path to avoid confusing the129 // analyzer.130 TSDs[I].assertLocked(/*BypassCheck=*/true);131 Str->append(" Shared TSD[%zu]:\n", I);132 TSDs[I].getSizeClassAllocator().getStats(Str);133 TSDs[I].unlock();134 }135 }136 137private:138 ALWAYS_INLINE TSD<Allocator> *getTSDAndLock() NO_THREAD_SAFETY_ANALYSIS {139 TSD<Allocator> *TSD = getCurrentTSD();140 DCHECK(TSD);141 // Try to lock the currently associated context.142 if (TSD->tryLock())143 return TSD;144 // If that fails, go down the slow path.145 if (TSDsArraySize == 1U) {146 // Only 1 TSD, not need to go any further.147 // The compiler will optimize this one way or the other.148 TSD->lock();149 return TSD;150 }151 return getTSDAndLockSlow(TSD);152 }153 154 ALWAYS_INLINE uptr *getTlsPtr() const {155#if SCUDO_HAS_PLATFORM_TLS_SLOT156 return reinterpret_cast<uptr *>(getPlatformAllocatorTlsSlot());157#else158 static thread_local uptr ThreadTSD;159 return &ThreadTSD;160#endif161 }162 163 static_assert(alignof(TSD<Allocator>) >= 2, "");164 165 ALWAYS_INLINE void setCurrentTSD(TSD<Allocator> *CurrentTSD) {166 *getTlsPtr() &= 1;167 *getTlsPtr() |= reinterpret_cast<uptr>(CurrentTSD);168 }169 170 ALWAYS_INLINE TSD<Allocator> *getCurrentTSD() {171 return reinterpret_cast<TSD<Allocator> *>(*getTlsPtr() & ~1ULL);172 }173 174 bool setNumberOfTSDs(u32 N) EXCLUDES(MutexTSDs) {175 ScopedLock L(MutexTSDs);176 if (N < NumberOfTSDs)177 return false;178 if (N > TSDsArraySize)179 N = TSDsArraySize;180 NumberOfTSDs = N;181 NumberOfCoPrimes = 0;182 // Compute all the coprimes of NumberOfTSDs. This will be used to walk the183 // array of TSDs in a random order. For details, see:184 // https://lemire.me/blog/2017/09/18/visiting-all-values-in-an-array-exactly-once-in-random-order/185 for (u32 I = 0; I < N; I++) {186 u32 A = I + 1;187 u32 B = N;188 // Find the GCD between I + 1 and N. If 1, they are coprimes.189 while (B != 0) {190 const u32 T = A;191 A = B;192 B = T % B;193 }194 if (A == 1)195 CoPrimes[NumberOfCoPrimes++] = I + 1;196 }197 return true;198 }199 200 void setDisableMemInit(bool B) {201 *getTlsPtr() &= ~1ULL;202 *getTlsPtr() |= B;203 }204 205 NOINLINE void initThread(Allocator *Instance) NO_THREAD_SAFETY_ANALYSIS {206 initOnceMaybe(Instance);207 // Initial context assignment is done in a plain round-robin fashion.208 const u32 Index = atomic_fetch_add(&CurrentIndex, 1U, memory_order_relaxed);209 setCurrentTSD(&TSDs[Index % NumberOfTSDs]);210 Instance->callPostInitCallback();211 }212 213 // TSDs is an array of locks which is not supported for marking thread-safety214 // capability.215 NOINLINE TSD<Allocator> *getTSDAndLockSlow(TSD<Allocator> *CurrentTSD)216 EXCLUDES(MutexTSDs) {217 // Use the Precedence of the current TSD as our random seed. Since we are218 // in the slow path, it means that tryLock failed, and as a result it's219 // very likely that said Precedence is non-zero.220 const u32 R = static_cast<u32>(CurrentTSD->getPrecedence());221 u32 N, Inc;222 {223 ScopedLock L(MutexTSDs);224 N = NumberOfTSDs;225 DCHECK_NE(NumberOfCoPrimes, 0U);226 Inc = CoPrimes[R % NumberOfCoPrimes];227 }228 if (N > 1U) {229 u32 Index = R % N;230 uptr LowestPrecedence = UINTPTR_MAX;231 TSD<Allocator> *CandidateTSD = nullptr;232 // Go randomly through at most 4 contexts and find a candidate.233 for (u32 I = 0; I < Min(4U, N); I++) {234 if (TSDs[Index].tryLock()) {235 setCurrentTSD(&TSDs[Index]);236 return &TSDs[Index];237 }238 const uptr Precedence = TSDs[Index].getPrecedence();239 // A 0 precedence here means another thread just locked this TSD.240 if (Precedence && Precedence < LowestPrecedence) {241 CandidateTSD = &TSDs[Index];242 LowestPrecedence = Precedence;243 }244 Index += Inc;245 if (Index >= N)246 Index -= N;247 }248 if (CandidateTSD) {249 CandidateTSD->lock();250 setCurrentTSD(CandidateTSD);251 return CandidateTSD;252 }253 }254 // Last resort, stick with the current one.255 CurrentTSD->lock();256 return CurrentTSD;257 }258 259 atomic_u32 CurrentIndex = {};260 u32 NumberOfTSDs GUARDED_BY(MutexTSDs) = 0;261 u32 NumberOfCoPrimes GUARDED_BY(MutexTSDs) = 0;262 u32 CoPrimes[TSDsArraySize] GUARDED_BY(MutexTSDs) = {};263 bool Initialized GUARDED_BY(Mutex) = false;264 HybridMutex Mutex;265 HybridMutex MutexTSDs;266 TSD<Allocator> TSDs[TSDsArraySize];267};268 269} // namespace scudo270 271#endif // SCUDO_TSD_SHARED_H_272