230 lines · c
1//===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a part of HWAddressSanitizer.10//11//===----------------------------------------------------------------------===//12 13// HwasanThreadList is a registry for live threads, as well as an allocator for14// HwasanThread objects and their stack history ring buffers. There are15// constraints on memory layout of the shadow region and CompactRingBuffer that16// are part of the ABI contract between compiler-rt and llvm.17//18// * Start of the shadow memory region is aligned to 2**kShadowBaseAlignment.19// * All stack ring buffers are located within (2**kShadowBaseAlignment)20// sized region below and adjacent to the shadow region.21// * Each ring buffer has a size of (2**N)*4096 where N is in [0, 7), and is22// aligned to twice its size. The value of N can be different for each buffer.23//24// These constrains guarantee that, given an address A of any element of the25// ring buffer,26// A_next = (A + sizeof(uptr)) & ~((1 << (N + 13)) - 1)27// is the address of the next element of that ring buffer (with wrap-around).28// And, with K = kShadowBaseAlignment,29// S = (A | ((1 << K) - 1)) + 130// (align up to kShadowBaseAlignment) is the start of the shadow region.31//32// These calculations are used in compiler instrumentation to update the ring33// buffer and obtain the base address of shadow using only two inputs: address34// of the current element of the ring buffer, and N (i.e. size of the ring35// buffer). Since the value of N is very limited, we pack both inputs into a36// single thread-local word as37// (1 << (N + 56)) | A38// See the implementation of class CompactRingBuffer, which is what is stored in39// said thread-local word.40//41// Note the unusual way of aligning up the address of the shadow:42// (A | ((1 << K) - 1)) + 143// It is only correct if A is not already equal to the shadow base address, but44// it saves 2 instructions on AArch64.45 46#include "hwasan.h"47#include "hwasan_allocator.h"48#include "hwasan_flags.h"49#include "hwasan_thread.h"50#include "sanitizer_common/sanitizer_thread_arg_retval.h"51 52namespace __hwasan {53 54static uptr RingBufferSize() {55 uptr desired_bytes = flags()->stack_history_size * sizeof(uptr);56 // FIXME: increase the limit to 8 once this bug is fixed:57 // https://bugs.llvm.org/show_bug.cgi?id=3903058 // Note that we *cannot* do that on Android, as the runtime will indefinitely59 // have to support code that is compiled with ashr, which only works with60 // shifts up to 6.61 for (int shift = 0; shift < 7; ++shift) {62 uptr size = 4096 * (1ULL << shift);63 if (size >= desired_bytes)64 return size;65 }66 Printf("stack history size too large: %d\n", flags()->stack_history_size);67 CHECK(0);68 return 0;69}70 71struct ThreadStats {72 uptr n_live_threads;73 uptr total_stack_size;74};75 76class SANITIZER_MUTEX HwasanThreadList {77 public:78 HwasanThreadList(uptr storage, uptr size)79 : free_space_(storage), free_space_end_(storage + size) {80 // [storage, storage + size) is used as a vector of81 // thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements.82 // Each element contains83 // * a ring buffer at offset 0,84 // * a Thread object at offset ring_buffer_size_.85 ring_buffer_size_ = RingBufferSize();86 thread_alloc_size_ =87 RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2);88 }89 90 Thread *CreateCurrentThread(const Thread::InitState *state = nullptr)91 SANITIZER_EXCLUDES(free_list_mutex_, live_list_mutex_) {92 Thread *t = nullptr;93 {94 SpinMutexLock l(&free_list_mutex_);95 if (!free_list_.empty()) {96 t = free_list_.back();97 free_list_.pop_back();98 }99 }100 if (t) {101 uptr start = (uptr)t - ring_buffer_size_;102 internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread));103 } else {104 t = AllocThread();105 }106 {107 SpinMutexLock l(&live_list_mutex_);108 live_list_.push_back(t);109 }110 t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_, state);111 AddThreadStats(t);112 return t;113 }114 115 void DontNeedThread(Thread *t) {116 uptr start = (uptr)t - ring_buffer_size_;117 ReleaseMemoryPagesToOS(start, start + thread_alloc_size_);118 }119 120 void RemoveThreadFromLiveList(Thread *t)121 SANITIZER_EXCLUDES(live_list_mutex_) {122 SpinMutexLock l(&live_list_mutex_);123 for (Thread *&t2 : live_list_)124 if (t2 == t) {125 // To remove t2, copy the last element of the list in t2's position, and126 // pop_back(). This works even if t2 is itself the last element.127 t2 = live_list_.back();128 live_list_.pop_back();129 return;130 }131 CHECK(0 && "thread not found in live list");132 }133 134 void ReleaseThread(Thread *t) SANITIZER_EXCLUDES(free_list_mutex_) {135 RemoveThreadStats(t);136 RemoveThreadFromLiveList(t);137 t->Destroy();138 DontNeedThread(t);139 SpinMutexLock l(&free_list_mutex_);140 free_list_.push_back(t);141 }142 143 Thread *GetThreadByBufferAddress(uptr p) {144 return (Thread *)(RoundDownTo(p, ring_buffer_size_ * 2) +145 ring_buffer_size_);146 }147 148 uptr MemoryUsedPerThread() {149 uptr res = sizeof(Thread) + ring_buffer_size_;150 if (auto sz = flags()->heap_history_size)151 res += HeapAllocationsRingBuffer::SizeInBytes(sz);152 return res;153 }154 155 template <class CB>156 void VisitAllLiveThreads(CB cb) SANITIZER_EXCLUDES(live_list_mutex_) {157 SpinMutexLock l(&live_list_mutex_);158 for (Thread *t : live_list_) cb(t);159 }160 161 template <class CB>162 Thread *FindThreadLocked(CB cb) SANITIZER_CHECK_LOCKED(live_list_mutex_) {163 CheckLocked();164 for (Thread *t : live_list_)165 if (cb(t))166 return t;167 return nullptr;168 }169 170 void AddThreadStats(Thread *t) SANITIZER_EXCLUDES(stats_mutex_) {171 SpinMutexLock l(&stats_mutex_);172 stats_.n_live_threads++;173 stats_.total_stack_size += t->stack_size();174 }175 176 void RemoveThreadStats(Thread *t) SANITIZER_EXCLUDES(stats_mutex_) {177 SpinMutexLock l(&stats_mutex_);178 stats_.n_live_threads--;179 stats_.total_stack_size -= t->stack_size();180 }181 182 ThreadStats GetThreadStats() SANITIZER_EXCLUDES(stats_mutex_) {183 SpinMutexLock l(&stats_mutex_);184 return stats_;185 }186 187 uptr GetRingBufferSize() const { return ring_buffer_size_; }188 189 void Lock() SANITIZER_ACQUIRE(live_list_mutex_) { live_list_mutex_.Lock(); }190 void CheckLocked() const SANITIZER_CHECK_LOCKED(live_list_mutex_) {191 live_list_mutex_.CheckLocked();192 }193 void Unlock() SANITIZER_RELEASE(live_list_mutex_) {194 live_list_mutex_.Unlock();195 }196 197 private:198 Thread *AllocThread() {199 SpinMutexLock l(&free_space_mutex_);200 uptr align = ring_buffer_size_ * 2;201 CHECK(IsAligned(free_space_, align));202 Thread *t = (Thread *)(free_space_ + ring_buffer_size_);203 free_space_ += thread_alloc_size_;204 CHECK_LE(free_space_, free_space_end_);205 return t;206 }207 208 SpinMutex free_space_mutex_;209 uptr free_space_;210 uptr free_space_end_;211 uptr ring_buffer_size_;212 uptr thread_alloc_size_;213 214 SpinMutex free_list_mutex_;215 InternalMmapVector<Thread *> free_list_216 SANITIZER_GUARDED_BY(free_list_mutex_);217 SpinMutex live_list_mutex_;218 InternalMmapVector<Thread *> live_list_219 SANITIZER_GUARDED_BY(live_list_mutex_);220 221 SpinMutex stats_mutex_;222 ThreadStats stats_ SANITIZER_GUARDED_BY(stats_mutex_);223};224 225void InitThreadList(uptr storage, uptr size);226HwasanThreadList &hwasanThreadList();227ThreadArgRetval &hwasanThreadArgRetval();228 229} // namespace __hwasan230