568 lines · cpp
1//===-- tsan_interceptors_mac.cpp -----------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a part of ThreadSanitizer (TSan), a race detector.10//11// Mac-specific interceptors.12//===----------------------------------------------------------------------===//13 14#include "sanitizer_common/sanitizer_platform.h"15#if SANITIZER_APPLE16 17# include <errno.h>18# include <libkern/OSAtomic.h>19# include <objc/objc-sync.h>20# include <os/lock.h>21# include <sys/ucontext.h>22 23# include "interception/interception.h"24# include "sanitizer_common/sanitizer_addrhashmap.h"25# include "tsan_interceptors.h"26# include "tsan_interface.h"27# include "tsan_interface_ann.h"28 29# if defined(__has_include) && __has_include(<xpc/xpc.h>)30# include <xpc/xpc.h>31# endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)32 33typedef long long_t;34 35extern "C" {36int getcontext(ucontext_t *ucp) __attribute__((returns_twice));37int setcontext(const ucontext_t *ucp);38}39 40namespace __tsan {41 42// The non-barrier versions of OSAtomic* functions are semantically mo_relaxed,43// but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are44// actually aliases of each other, and we cannot have different interceptors for45// them, because they're actually the same function. Thus, we have to stay46// conservative and treat the non-barrier versions as mo_acq_rel.47static constexpr morder kMacOrderBarrier = mo_acq_rel;48static constexpr morder kMacOrderNonBarrier = mo_acq_rel;49static constexpr morder kMacFailureOrder = mo_relaxed;50 51# define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \52 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \53 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \54 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \55 }56 57# define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, \58 mo) \59 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \60 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \61 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \62 }63 64# define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \65 mo) \66 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \67 SCOPED_TSAN_INTERCEPTOR(f, ptr); \68 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \69 }70 71# define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \72 mo) \73 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \74 SCOPED_TSAN_INTERCEPTOR(f, ptr); \75 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \76 }77 78# define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \79 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \80 kMacOrderNonBarrier) \81 m(int32_t, int32_t, a32, f##32##Barrier, \82 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \83 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \84 kMacOrderNonBarrier) \85 m(int64_t, int64_t, a64, f##64##Barrier, \86 __tsan_atomic64_##tsan_atomic_f, kMacOrderBarrier)87 88# define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \89 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \90 kMacOrderNonBarrier) \91 m(int32_t, uint32_t, a32, f##32##Barrier, \92 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \93 m_orig(int32_t, uint32_t, a32, f##32##Orig, \94 __tsan_atomic32_##tsan_atomic_f, kMacOrderNonBarrier) \95 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \96 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier)97 98# pragma clang diagnostic push // OSAtomic* deprecation99# pragma clang diagnostic ignored "-Wdeprecated-declarations"100OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add,101 OSATOMIC_INTERCEPTOR_PLUS_X)102OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add,103 OSATOMIC_INTERCEPTOR_PLUS_1)104OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub,105 OSATOMIC_INTERCEPTOR_MINUS_1)106OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X,107 OSATOMIC_INTERCEPTOR)108OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and,109 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)110OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor,111 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR)112# pragma clang diagnostic pop // OSAtomic* deprecation113 114# define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \115 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \116 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \117 return tsan_atomic_f##_compare_exchange_strong( \118 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \119 kMacOrderNonBarrier, kMacFailureOrder); \120 } \121 \122 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \123 t volatile *ptr) { \124 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \125 return tsan_atomic_f##_compare_exchange_strong( \126 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \127 kMacOrderBarrier, kMacFailureOrder); \128 }129 130# pragma clang diagnostic push // OSAtomicCompareAndSwap* deprecation131# pragma clang diagnostic ignored "-Wdeprecated-declarations"132OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int)133OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64,134 long_t)135OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64,136 void *)137OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32,138 int32_t)139OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64,140 int64_t)141# pragma clang diagnostic pop // OSAtomicCompareAndSwap* deprecation142 143# define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \144 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \145 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \146 volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \147 char bit = 0x80u >> (n & 7); \148 char mask = clear ? ~bit : bit; \149 char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \150 return orig_byte & bit; \151 }152 153# define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \154 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \155 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier)156 157# pragma clang diagnostic push // OSAtomicTestAnd* deprecation158# pragma clang diagnostic ignored "-Wdeprecated-declarations"159OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false)160OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and,161 true)162# pragma clang diagnostic pop // OSAtomicTestAnd* deprecation163 164TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item,165 size_t offset) {166 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset);167 __tsan_release(item);168 REAL(OSAtomicEnqueue)(list, item, offset);169}170 171TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) {172 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset);173 void *item = REAL(OSAtomicDequeue)(list, offset);174 if (item)175 __tsan_acquire(item);176 return item;177}178 179// OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X.180# if !SANITIZER_IOS181 182TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item,183 size_t offset) {184 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset);185 __tsan_release(item);186 REAL(OSAtomicFifoEnqueue)(list, item, offset);187}188 189TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list,190 size_t offset) {191 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset);192 void *item = REAL(OSAtomicFifoDequeue)(list, offset);193 if (item)194 __tsan_acquire(item);195 return item;196}197 198# endif199 200// If `OSSPINLOCK_USE_INLINED=1` is set, then SDK headers don't declare these201// as functions, but macros that call non-deprecated APIs. Undefine these202// macros so they don't interfere with the interceptor machinery.203# undef OSSpinLockLock204# undef OSSpinLockTry205# undef OSSpinLockUnlock206 207# pragma clang diagnostic push // OSSpinLock* deprecation208# pragma clang diagnostic ignored "-Wdeprecated-declarations"209 210TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) {211 CHECK(!cur_thread()->is_dead);212 if (!cur_thread()->is_inited) {213 return REAL(OSSpinLockLock)(lock);214 }215 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock);216 REAL(OSSpinLockLock)(lock);217 Acquire(thr, pc, (uptr)lock);218}219 220TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) {221 CHECK(!cur_thread()->is_dead);222 if (!cur_thread()->is_inited) {223 return REAL(OSSpinLockTry)(lock);224 }225 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock);226 bool result = REAL(OSSpinLockTry)(lock);227 if (result)228 Acquire(thr, pc, (uptr)lock);229 return result;230}231 232TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) {233 CHECK(!cur_thread()->is_dead);234 if (!cur_thread()->is_inited) {235 return REAL(OSSpinLockUnlock)(lock);236 }237 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock);238 Release(thr, pc, (uptr)lock);239 REAL(OSSpinLockUnlock)(lock);240}241# pragma clang diagnostic pop // OSSpinLock* deprecation242 243TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) {244 CHECK(!cur_thread()->is_dead);245 if (!cur_thread()->is_inited) {246 return REAL(os_lock_lock)(lock);247 }248 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock);249 REAL(os_lock_lock)(lock);250 Acquire(thr, pc, (uptr)lock);251}252 253TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) {254 CHECK(!cur_thread()->is_dead);255 if (!cur_thread()->is_inited) {256 return REAL(os_lock_trylock)(lock);257 }258 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock);259 bool result = REAL(os_lock_trylock)(lock);260 if (result)261 Acquire(thr, pc, (uptr)lock);262 return result;263}264 265TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) {266 CHECK(!cur_thread()->is_dead);267 if (!cur_thread()->is_inited) {268 return REAL(os_lock_unlock)(lock);269 }270 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock);271 Release(thr, pc, (uptr)lock);272 REAL(os_lock_unlock)(lock);273}274 275TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) {276 if (!cur_thread()->is_inited || cur_thread()->is_dead) {277 return REAL(os_unfair_lock_lock)(lock);278 }279 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock);280 REAL(os_unfair_lock_lock)(lock);281 Acquire(thr, pc, (uptr)lock);282}283 284// os_unfair_lock_lock_with_flags was introduced in macOS 15285# if defined(__MAC_15_0) || defined(__IPHONE_18_0) || defined(__TVOS_18_0) || \286 defined(__VISIONOS_2_0) || defined(__WATCHOS_11_0)287# pragma clang diagnostic push288# pragma clang diagnostic ignored "-Wunguarded-availability-new"289// We're just intercepting this - if it doesn't exist on the platform, then the290// process shouldn't have called it in the first place.291TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_flags, os_unfair_lock_t lock,292 os_unfair_lock_flags_t flags) {293 if (!cur_thread()->is_inited || cur_thread()->is_dead) {294 return REAL(os_unfair_lock_lock_with_flags)(lock, flags);295 }296 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_flags, lock, flags);297 REAL(os_unfair_lock_lock_with_flags)(lock, flags);298 Acquire(thr, pc, (uptr)lock);299}300# pragma clang diagnostic pop301# endif302 303TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock,304 u32 options) {305 if (!cur_thread()->is_inited || cur_thread()->is_dead) {306 return REAL(os_unfair_lock_lock_with_options)(lock, options);307 }308 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options);309 REAL(os_unfair_lock_lock_with_options)(lock, options);310 Acquire(thr, pc, (uptr)lock);311}312 313TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) {314 if (!cur_thread()->is_inited || cur_thread()->is_dead) {315 return REAL(os_unfair_lock_trylock)(lock);316 }317 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock);318 bool result = REAL(os_unfair_lock_trylock)(lock);319 if (result)320 Acquire(thr, pc, (uptr)lock);321 return result;322}323 324TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) {325 if (!cur_thread()->is_inited || cur_thread()->is_dead) {326 return REAL(os_unfair_lock_unlock)(lock);327 }328 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock);329 Release(thr, pc, (uptr)lock);330 REAL(os_unfair_lock_unlock)(lock);331}332 333# if defined(__has_include) && __has_include(<xpc/xpc.h>)334 335TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler,336 xpc_connection_t connection, xpc_handler_t handler) {337 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection,338 handler);339 Release(thr, pc, (uptr)connection);340 xpc_handler_t new_handler = ^(xpc_object_t object) {341 {342 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler);343 Acquire(thr, pc, (uptr)connection);344 }345 handler(object);346 };347 REAL(xpc_connection_set_event_handler)(connection, new_handler);348}349 350TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection,351 dispatch_block_t barrier) {352 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier);353 Release(thr, pc, (uptr)connection);354 dispatch_block_t new_barrier = ^() {355 {356 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier);357 Acquire(thr, pc, (uptr)connection);358 }359 barrier();360 };361 REAL(xpc_connection_send_barrier)(connection, new_barrier);362}363 364TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply,365 xpc_connection_t connection, xpc_object_t message,366 dispatch_queue_t replyq, xpc_handler_t handler) {367 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection,368 message, replyq, handler);369 Release(thr, pc, (uptr)connection);370 xpc_handler_t new_handler = ^(xpc_object_t object) {371 {372 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply);373 Acquire(thr, pc, (uptr)connection);374 }375 handler(object);376 };377 REAL(xpc_connection_send_message_with_reply)378 (connection, message, replyq, new_handler);379}380 381TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) {382 SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection);383 Release(thr, pc, (uptr)connection);384 REAL(xpc_connection_cancel)(connection);385}386 387# endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>)388 389// Determines whether the Obj-C object pointer is a tagged pointer. Tagged390// pointers encode the object data directly in their pointer bits and do not391// have an associated memory allocation. The Obj-C runtime uses tagged pointers392// to transparently optimize small objects.393static bool IsTaggedObjCPointer(id obj) {394 const uptr kPossibleTaggedBits = 0x8000000000000001ull;395 return ((uptr)obj & kPossibleTaggedBits) != 0;396}397 398// Returns an address which can be used to inform TSan about synchronization399// points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid400// address in the process space. We do a small allocation here to obtain a401// stable address (the array backing the hash map can change). The memory is402// never free'd (leaked) and allocation and locking are slow, but this code only403// runs for @synchronized with tagged pointers, which is very rare.404static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) {405 typedef AddrHashMap<uptr, 5> Map;406 static Map Addresses;407 Map::Handle h(&Addresses, addr);408 if (h.created()) {409 ThreadIgnoreBegin(thr, pc);410 *h = (uptr)user_alloc(thr, pc, /*size=*/1);411 ThreadIgnoreEnd(thr);412 }413 return *h;414}415 416// Returns an address on which we can synchronize given an Obj-C object pointer.417// For normal object pointers, this is just the address of the object in memory.418// Tagged pointers are not backed by an actual memory allocation, so we need to419// synthesize a valid address.420static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) {421 if (IsTaggedObjCPointer(obj))422 return GetOrCreateSyncAddress((uptr)obj, thr, pc);423 return (uptr)obj;424}425 426TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) {427 SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj);428 if (!obj)429 return REAL(objc_sync_enter)(obj);430 uptr addr = SyncAddressForObjCObject(obj, thr, pc);431 MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant);432 int result = REAL(objc_sync_enter)(obj);433 CHECK_EQ(result, OBJC_SYNC_SUCCESS);434 MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant);435 return result;436}437 438TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) {439 SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj);440 if (!obj)441 return REAL(objc_sync_exit)(obj);442 uptr addr = SyncAddressForObjCObject(obj, thr, pc);443 MutexUnlock(thr, pc, addr);444 int result = REAL(objc_sync_exit)(obj);445 if (result != OBJC_SYNC_SUCCESS)446 MutexInvalidAccess(thr, pc, addr);447 return result;448}449 450TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) {451 {452 SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp);453 }454 // Because of swapcontext() semantics we have no option but to copy its455 // implementation here456 if (!oucp || !ucp) {457 errno = EINVAL;458 return -1;459 }460 ThreadState *thr = cur_thread();461 const int UCF_SWAPPED = 0x80000000;462 oucp->uc_onstack &= ~UCF_SWAPPED;463 thr->ignore_interceptors++;464 int ret = getcontext(oucp);465 if (!(oucp->uc_onstack & UCF_SWAPPED)) {466 thr->ignore_interceptors--;467 if (!ret) {468 oucp->uc_onstack |= UCF_SWAPPED;469 ret = setcontext(ucp);470 }471 }472 return ret;473}474 475// On macOS, libc++ is always linked dynamically, so intercepting works the476// usual way.477# define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR478 479namespace {480struct fake_shared_weak_count {481 volatile a64 shared_owners;482 volatile a64 shared_weak_owners;483 virtual void _unused_0x0() = 0;484 virtual void _unused_0x8() = 0;485 virtual void on_zero_shared() = 0;486 virtual void _unused_0x18() = 0;487 virtual void on_zero_shared_weak() = 0;488 virtual ~fake_shared_weak_count() = 0; // suppress -Wnon-virtual-dtor489};490} // namespace491 492// The following code adds libc++ interceptors for:493// void __shared_weak_count::__release_shared() _NOEXCEPT;494// bool __shared_count::__release_shared() _NOEXCEPT;495// Shared and weak pointers in C++ maintain reference counts via atomics in496// libc++.dylib, which are TSan-invisible, and this leads to false positives in497// destructor code. These interceptors re-implements the whole functions so that498// the mo_acq_rel semantics of the atomic decrement are visible.499//500// Unfortunately, the interceptors cannot simply Acquire/Release some sync501// object and call the original function, because it would have a race between502// the sync and the destruction of the object. Calling both under a lock will503// not work because the destructor can invoke this interceptor again (and even504// in a different thread, so recursive locks don't help).505 506STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv,507 fake_shared_weak_count *o) {508 if (!flags()->shared_ptr_interceptor)509 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o);510 511 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv,512 o);513 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {514 Acquire(thr, pc, (uptr)&o->shared_owners);515 o->on_zero_shared();516 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) ==517 0) {518 Acquire(thr, pc, (uptr)&o->shared_weak_owners);519 o->on_zero_shared_weak();520 }521 }522}523 524STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv,525 fake_shared_weak_count *o) {526 if (!flags()->shared_ptr_interceptor)527 return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o);528 529 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o);530 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) {531 Acquire(thr, pc, (uptr)&o->shared_owners);532 o->on_zero_shared();533 return true;534 }535 return false;536}537 538namespace {539struct call_once_callback_args {540 void (*orig_func)(void *arg);541 void *orig_arg;542 void *flag;543};544 545void call_once_callback_wrapper(void *arg) {546 call_once_callback_args *new_args = (call_once_callback_args *)arg;547 new_args->orig_func(new_args->orig_arg);548 __tsan_release(new_args->flag);549}550} // namespace551 552// This adds a libc++ interceptor for:553// void __call_once(volatile unsigned long&, void*, void(*)(void*));554// C++11 call_once is implemented via an internal function __call_once which is555// inside libc++.dylib, and the atomic release store inside it is thus556// TSan-invisible. To avoid false positives, this interceptor wraps the callback557// function and performs an explicit Release after the user code has run.558STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag,559 void *arg, void (*func)(void *arg)) {560 call_once_callback_args new_args = {func, arg, flag};561 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args,562 call_once_callback_wrapper);563}564 565} // namespace __tsan566 567#endif // SANITIZER_APPLE568