815 lines · cpp
1//===-- tsan_interceptors_libdispatch.cpp ---------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a part of ThreadSanitizer (TSan), a race detector.10//11// Support for intercepting libdispatch (GCD).12//===----------------------------------------------------------------------===//13 14#include "sanitizer_common/sanitizer_common.h"15#include "interception/interception.h"16#include "tsan_interceptors.h"17#include "tsan_rtl.h"18 19#include "BlocksRuntime/Block.h"20#include "tsan_dispatch_defs.h"21 22#if SANITIZER_APPLE23# include <Availability.h>24#endif25 26namespace __tsan {27 typedef u16 uint16_t;28 29typedef struct {30 dispatch_queue_t queue;31 void *orig_context;32 dispatch_function_t orig_work;33 bool free_context_in_callback;34 bool submitted_synchronously;35 bool is_barrier_block;36 uptr non_queue_sync_object;37} block_context_t;38 39// The offsets of different fields of the dispatch_queue_t structure, exported40// by libdispatch.dylib.41extern "C" struct dispatch_queue_offsets_s {42 const uint16_t dqo_version;43 const uint16_t dqo_label;44 const uint16_t dqo_label_size;45 const uint16_t dqo_flags;46 const uint16_t dqo_flags_size;47 const uint16_t dqo_serialnum;48 const uint16_t dqo_serialnum_size;49 const uint16_t dqo_width;50 const uint16_t dqo_width_size;51 const uint16_t dqo_running;52 const uint16_t dqo_running_size;53 const uint16_t dqo_suspend_cnt;54 const uint16_t dqo_suspend_cnt_size;55 const uint16_t dqo_target_queue;56 const uint16_t dqo_target_queue_size;57 const uint16_t dqo_priority;58 const uint16_t dqo_priority_size;59} dispatch_queue_offsets;60 61static bool IsQueueSerial(dispatch_queue_t q) {62 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);63 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);64 CHECK_NE(width, 0);65 return width == 1;66}67 68static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) {69 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8);70 dispatch_queue_t tq = *(71 dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue);72 return tq;73}74 75static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) {76 dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source);77 CHECK_NE(tq, 0);78 return tq;79}80 81static block_context_t *AllocContext(ThreadState *thr, uptr pc,82 dispatch_queue_t queue, void *orig_context,83 dispatch_function_t orig_work) {84 block_context_t *new_context =85 (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t));86 new_context->queue = queue;87 new_context->orig_context = orig_context;88 new_context->orig_work = orig_work;89 new_context->free_context_in_callback = true;90 new_context->submitted_synchronously = false;91 new_context->is_barrier_block = false;92 new_context->non_queue_sync_object = 0;93 return new_context;94}95 96#define GET_QUEUE_SYNC_VARS(context, q) \97 bool is_queue_serial = q && IsQueueSerial(q); \98 uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \99 uptr serial_sync = (uptr)sync_ptr; \100 uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \101 bool serial_task = context->is_barrier_block || is_queue_serial102 103static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc,104 block_context_t *context) {105 uptr submit_sync = (uptr)context;106 Acquire(thr, pc, submit_sync);107 108 dispatch_queue_t q = context->queue;109 do {110 GET_QUEUE_SYNC_VARS(context, q);111 if (serial_sync) Acquire(thr, pc, serial_sync);112 if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync);113 114 if (q) q = GetTargetQueueFromQueue(q);115 } while (q);116}117 118static void dispatch_sync_post_execute(ThreadState *thr, uptr pc,119 block_context_t *context) {120 uptr submit_sync = (uptr)context;121 if (context->submitted_synchronously) Release(thr, pc, submit_sync);122 123 dispatch_queue_t q = context->queue;124 do {125 GET_QUEUE_SYNC_VARS(context, q);126 if (serial_task && serial_sync) Release(thr, pc, serial_sync);127 if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync);128 129 if (q) q = GetTargetQueueFromQueue(q);130 } while (q);131}132 133static void dispatch_callback_wrap(void *param) {134 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap);135 block_context_t *context = (block_context_t *)param;136 137 dispatch_sync_pre_execute(thr, pc, context);138 139 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();140 context->orig_work(context->orig_context);141 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();142 143 dispatch_sync_post_execute(thr, pc, context);144 145 if (context->free_context_in_callback) user_free(thr, pc, context);146}147 148static void invoke_block(void *param) {149 dispatch_block_t block = (dispatch_block_t)param;150 block();151}152 153static void invoke_and_release_block(void *param) {154 dispatch_block_t block = (dispatch_block_t)param;155 block();156 Block_release(block);157}158 159#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \160 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \161 SCOPED_TSAN_INTERCEPTOR(name, q, block); \162 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \163 dispatch_block_t heap_block = Block_copy(block); \164 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \165 block_context_t *new_context = \166 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \167 new_context->is_barrier_block = barrier; \168 Release(thr, pc, (uptr)new_context); \169 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \170 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \171 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \172 }173 174#define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \175 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \176 DISPATCH_NOESCAPE dispatch_block_t block) { \177 SCOPED_TSAN_INTERCEPTOR(name, q, block); \178 block_context_t new_context = { \179 q, block, &invoke_block, false, true, barrier, 0}; \180 Release(thr, pc, (uptr)&new_context); \181 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \182 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \183 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \184 Acquire(thr, pc, (uptr)&new_context); \185 }186 187#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \188 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \189 dispatch_function_t work) { \190 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \191 block_context_t *new_context = \192 AllocContext(thr, pc, q, context, work); \193 new_context->is_barrier_block = barrier; \194 Release(thr, pc, (uptr)new_context); \195 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \196 REAL(name)(q, new_context, dispatch_callback_wrap); \197 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \198 }199 200#define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \201 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \202 dispatch_function_t work) { \203 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \204 block_context_t new_context = { \205 q, context, work, false, true, barrier, 0}; \206 Release(thr, pc, (uptr)&new_context); \207 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \208 REAL(name)(q, &new_context, dispatch_callback_wrap); \209 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \210 Acquire(thr, pc, (uptr)&new_context); \211 }212 213#define DISPATCH_INTERCEPT(name, barrier) \214 DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \215 DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \216 DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \217 DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier)218 219// We wrap dispatch_async, dispatch_sync and friends where we allocate a new220// context, which is used to synchronize (we release the context before221// submitting, and the callback acquires it before executing the original222// callback).223DISPATCH_INTERCEPT(dispatch, false)224DISPATCH_INTERCEPT(dispatch_barrier, true)225 226// dispatch_async_and_wait() and friends were introduced in macOS 10.14.227// Linking of these interceptors fails when using an older SDK.228#if !SANITIZER_APPLE || defined(__MAC_10_14)229// macOS 10.14 is greater than our minimal deployment target. To ensure we230// generate a weak reference so the TSan dylib continues to work on older231// systems, we need to forward declare the intercepted functions as "weak232// imports". Note that this file is multi-platform, so we cannot include the233// actual header file (#include <dispatch/dispatch.h>).234SANITIZER_WEAK_IMPORT void dispatch_async_and_wait(235 dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block);236SANITIZER_WEAK_IMPORT void dispatch_async_and_wait_f(237 dispatch_queue_t queue, void *context, dispatch_function_t work);238SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait(239 dispatch_queue_t queue, DISPATCH_NOESCAPE dispatch_block_t block);240SANITIZER_WEAK_IMPORT void dispatch_barrier_async_and_wait_f(241 dispatch_queue_t queue, void *context, dispatch_function_t work);242 243DISPATCH_INTERCEPT_SYNC_F(dispatch_async_and_wait_f, false)244DISPATCH_INTERCEPT_SYNC_B(dispatch_async_and_wait, false)245DISPATCH_INTERCEPT_SYNC_F(dispatch_barrier_async_and_wait_f, true)246DISPATCH_INTERCEPT_SYNC_B(dispatch_barrier_async_and_wait, true)247#endif248 249 250DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when,251 dispatch_queue_t queue, void *context, dispatch_function_t work)252 253TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when,254 dispatch_queue_t queue, dispatch_block_t block) {255 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block);256 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();257 dispatch_block_t heap_block = Block_copy(block);258 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();259 block_context_t *new_context =260 AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block);261 Release(thr, pc, (uptr)new_context);262 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();263 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap);264 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();265}266 267TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,268 dispatch_queue_t queue, void *context,269 dispatch_function_t work) {270 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work);271 WRAP(dispatch_after)(when, queue, ^(void) {272 work(context);273 });274}275 276// GCD's dispatch_once implementation has a fast path that contains a racy read277// and it's inlined into user's code. Furthermore, this fast path doesn't278// establish a proper happens-before relations between the initialization and279// code following the call to dispatch_once. We could deal with this in280// instrumented code, but there's not much we can do about it in system281// libraries. Let's disable the fast path (by never storing the value ~0 to282// predicate), so the interceptor is always called, and let's add proper release283// and acquire semantics. Since TSan does not see its own atomic stores, the284// race on predicate won't be reported - the only accesses to it that TSan sees285// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is286// both a macro and a real function, we want to intercept the function, so we287// need to undefine the macro.288#undef dispatch_once289TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,290 DISPATCH_NOESCAPE dispatch_block_t block) {291 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block);292 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);293 u32 v = atomic_load(a, memory_order_acquire);294 if (v == 0 &&295 atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {296 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();297 block();298 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();299 Release(thr, pc, (uptr)a);300 atomic_store(a, 2, memory_order_release);301 } else {302 while (v != 2) {303 internal_sched_yield();304 v = atomic_load(a, memory_order_acquire);305 }306 Acquire(thr, pc, (uptr)a);307 }308}309 310#undef dispatch_once_f311TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,312 void *context, dispatch_function_t function) {313 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function);314 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();315 WRAP(dispatch_once)(predicate, ^(void) {316 function(context);317 });318 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();319}320 321TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,322 dispatch_semaphore_t dsema) {323 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);324 Release(thr, pc, (uptr)dsema);325 return REAL(dispatch_semaphore_signal)(dsema);326}327 328TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,329 dispatch_time_t timeout) {330 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);331 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);332 if (result == 0) Acquire(thr, pc, (uptr)dsema);333 return result;334}335 336TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,337 dispatch_time_t timeout) {338 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);339 long_t result = REAL(dispatch_group_wait)(group, timeout);340 if (result == 0) Acquire(thr, pc, (uptr)group);341 return result;342}343 344// Used, but not intercepted.345extern "C" void dispatch_group_enter(dispatch_group_t group);346 347TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {348 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);349 // Acquired in the group notification callback in dispatch_group_notify[_f].350 Release(thr, pc, (uptr)group);351 REAL(dispatch_group_leave)(group);352}353 354TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,355 dispatch_queue_t queue, dispatch_block_t block) {356 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);357 dispatch_retain(group);358 dispatch_group_enter(group);359 __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block);360 WRAP(dispatch_async)(queue, ^(void) {361 block_copy();362 Block_release(block_copy);363 WRAP(dispatch_group_leave)(group);364 dispatch_release(group);365 });366}367 368TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,369 dispatch_queue_t queue, void *context,370 dispatch_function_t work) {371 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);372 dispatch_retain(group);373 dispatch_group_enter(group);374 WRAP(dispatch_async)(queue, ^(void) {375 work(context);376 WRAP(dispatch_group_leave)(group);377 dispatch_release(group);378 });379}380 381DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group,382 dispatch_queue_t q, void *context, dispatch_function_t work)383 384TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,385 dispatch_queue_t q, dispatch_block_t block) {386 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);387 388 // To make sure the group is still available in the callback (otherwise389 // it can be already destroyed). Will be released in the callback.390 dispatch_retain(group);391 392 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();393 dispatch_block_t heap_block = Block_copy(^(void) {394 {395 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback);396 // Released when leaving the group (dispatch_group_leave).397 Acquire(thr, pc, (uptr)group);398 }399 dispatch_release(group);400 block();401 });402 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();403 block_context_t *new_context =404 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);405 new_context->is_barrier_block = true;406 Release(thr, pc, (uptr)new_context);407 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap);408}409 410TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,411 dispatch_queue_t q, void *context, dispatch_function_t work) {412 WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); });413}414 415TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler,416 dispatch_source_t source, dispatch_block_t handler) {417 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler);418 if (handler == nullptr)419 return REAL(dispatch_source_set_event_handler)(source, nullptr);420 dispatch_queue_t q = GetTargetQueueFromSource(source);421 __block block_context_t new_context = {422 q, handler, &invoke_block, false, false, false, 0 };423 dispatch_block_t new_handler = Block_copy(^(void) {424 new_context.orig_context = handler; // To explicitly capture "handler".425 dispatch_callback_wrap(&new_context);426 });427 uptr submit_sync = (uptr)&new_context;428 Release(thr, pc, submit_sync);429 REAL(dispatch_source_set_event_handler)(source, new_handler);430 Block_release(new_handler);431}432 433TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f,434 dispatch_source_t source, dispatch_function_t handler) {435 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler);436 if (handler == nullptr)437 return REAL(dispatch_source_set_event_handler)(source, nullptr);438 dispatch_block_t block = ^(void) {439 handler(dispatch_get_context(source));440 };441 WRAP(dispatch_source_set_event_handler)(source, block);442}443 444TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler,445 dispatch_source_t source, dispatch_block_t handler) {446 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler);447 if (handler == nullptr)448 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);449 dispatch_queue_t q = GetTargetQueueFromSource(source);450 __block block_context_t new_context = {451 q, handler, &invoke_block, false, false, false, 0};452 dispatch_block_t new_handler = Block_copy(^(void) {453 new_context.orig_context = handler; // To explicitly capture "handler".454 dispatch_callback_wrap(&new_context);455 });456 uptr submit_sync = (uptr)&new_context;457 Release(thr, pc, submit_sync);458 REAL(dispatch_source_set_cancel_handler)(source, new_handler);459 Block_release(new_handler);460}461 462TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f,463 dispatch_source_t source, dispatch_function_t handler) {464 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source,465 handler);466 if (handler == nullptr)467 return REAL(dispatch_source_set_cancel_handler)(source, nullptr);468 dispatch_block_t block = ^(void) {469 handler(dispatch_get_context(source));470 };471 WRAP(dispatch_source_set_cancel_handler)(source, block);472}473 474TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler,475 dispatch_source_t source, dispatch_block_t handler) {476 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source,477 handler);478 if (handler == nullptr)479 return REAL(dispatch_source_set_registration_handler)(source, nullptr);480 dispatch_queue_t q = GetTargetQueueFromSource(source);481 __block block_context_t new_context = {482 q, handler, &invoke_block, false, false, false, 0};483 dispatch_block_t new_handler = Block_copy(^(void) {484 new_context.orig_context = handler; // To explicitly capture "handler".485 dispatch_callback_wrap(&new_context);486 });487 uptr submit_sync = (uptr)&new_context;488 Release(thr, pc, submit_sync);489 REAL(dispatch_source_set_registration_handler)(source, new_handler);490 Block_release(new_handler);491}492 493TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f,494 dispatch_source_t source, dispatch_function_t handler) {495 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source,496 handler);497 if (handler == nullptr)498 return REAL(dispatch_source_set_registration_handler)(source, nullptr);499 dispatch_block_t block = ^(void) {500 handler(dispatch_get_context(source));501 };502 WRAP(dispatch_source_set_registration_handler)(source, block);503}504 505TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations,506 dispatch_queue_t queue,507 DISPATCH_NOESCAPE void (^block)(size_t)) {508 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block);509 510 u8 sync1, sync2;511 uptr parent_to_child_sync = (uptr)&sync1;512 uptr child_to_parent_sync = (uptr)&sync2;513 514 Release(thr, pc, parent_to_child_sync);515 void (^new_block)(size_t) = ^(size_t iteration) {516 SCOPED_INTERCEPTOR_RAW(dispatch_apply);517 Acquire(thr, pc, parent_to_child_sync);518 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();519 block(iteration);520 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();521 Release(thr, pc, child_to_parent_sync);522 };523 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();524 REAL(dispatch_apply)(iterations, queue, new_block);525 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();526 Acquire(thr, pc, child_to_parent_sync);527}528 529static void invoke_block_iteration(void *param, size_t iteration) {530 auto block = (void (^)(size_t)) param;531 block(iteration);532}533 534TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations,535 dispatch_queue_t queue, void *context,536 void (*work)(void *, size_t)) {537 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work);538 539 // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch540 // implements dispatch_apply in terms of dispatch_apply_f.541 u8 sync1, sync2;542 uptr parent_to_child_sync = (uptr)&sync1;543 uptr child_to_parent_sync = (uptr)&sync2;544 545 Release(thr, pc, parent_to_child_sync);546 void (^new_block)(size_t) = ^(size_t iteration) {547 SCOPED_INTERCEPTOR_RAW(dispatch_apply_f);548 Acquire(thr, pc, parent_to_child_sync);549 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();550 work(context, iteration);551 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();552 Release(thr, pc, child_to_parent_sync);553 };554 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();555 REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration);556 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();557 Acquire(thr, pc, child_to_parent_sync);558}559 560DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)561DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, SIZE_T sz)562 563TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer,564 size_t size, dispatch_queue_t q, dispatch_block_t destructor) {565 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor);566 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT))567 return REAL(dispatch_data_create)(buffer, size, q, destructor);568 569 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE)570 destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); };571 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP)572 destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); };573 574 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();575 dispatch_block_t heap_block = Block_copy(destructor);576 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();577 block_context_t *new_context =578 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);579 uptr submit_sync = (uptr)new_context;580 Release(thr, pc, submit_sync);581 return REAL(dispatch_data_create)(buffer, size, q, ^(void) {582 dispatch_callback_wrap(new_context);583 });584}585 586typedef void (^fd_handler_t)(dispatch_data_t data, int error);587typedef void (^cleanup_handler_t)(int error);588 589TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length,590 dispatch_queue_t q, fd_handler_t h) {591 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h);592 __block block_context_t new_context = {593 q, nullptr, &invoke_block, false, false, false, 0};594 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {595 new_context.orig_context = ^(void) {596 h(data, error);597 };598 dispatch_callback_wrap(&new_context);599 });600 uptr submit_sync = (uptr)&new_context;601 Release(thr, pc, submit_sync);602 REAL(dispatch_read)(fd, length, q, new_h);603 Block_release(new_h);604}605 606TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data,607 dispatch_queue_t q, fd_handler_t h) {608 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h);609 __block block_context_t new_context = {610 q, nullptr, &invoke_block, false, false, false, 0};611 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) {612 new_context.orig_context = ^(void) {613 h(data, error);614 };615 dispatch_callback_wrap(&new_context);616 });617 uptr submit_sync = (uptr)&new_context;618 Release(thr, pc, submit_sync);619 REAL(dispatch_write)(fd, data, q, new_h);620 Block_release(new_h);621}622 623TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset,624 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) {625 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h);626 __block block_context_t new_context = {627 q, nullptr, &invoke_block, false, false, false, 0};628 dispatch_io_handler_t new_h =629 Block_copy(^(bool done, dispatch_data_t data, int error) {630 new_context.orig_context = ^(void) {631 h(done, data, error);632 };633 dispatch_callback_wrap(&new_context);634 });635 uptr submit_sync = (uptr)&new_context;636 Release(thr, pc, submit_sync);637 REAL(dispatch_io_read)(channel, offset, length, q, new_h);638 Block_release(new_h);639}640 641TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset,642 dispatch_data_t data, dispatch_queue_t q,643 dispatch_io_handler_t h) {644 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h);645 __block block_context_t new_context = {646 q, nullptr, &invoke_block, false, false, false, 0};647 dispatch_io_handler_t new_h =648 Block_copy(^(bool done, dispatch_data_t data, int error) {649 new_context.orig_context = ^(void) {650 h(done, data, error);651 };652 dispatch_callback_wrap(&new_context);653 });654 uptr submit_sync = (uptr)&new_context;655 Release(thr, pc, submit_sync);656 REAL(dispatch_io_write)(channel, offset, data, q, new_h);657 Block_release(new_h);658}659 660TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel,661 dispatch_block_t barrier) {662 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier);663 __block block_context_t new_context = {664 nullptr, nullptr, &invoke_block, false, false, false, 0};665 new_context.non_queue_sync_object = (uptr)channel;666 new_context.is_barrier_block = true;667 dispatch_block_t new_block = Block_copy(^(void) {668 new_context.orig_context = ^(void) {669 barrier();670 };671 dispatch_callback_wrap(&new_context);672 });673 uptr submit_sync = (uptr)&new_context;674 Release(thr, pc, submit_sync);675 REAL(dispatch_io_barrier)(channel, new_block);676 Block_release(new_block);677}678 679TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type,680 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) {681 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h);682 __block dispatch_io_t new_channel = nullptr;683 __block block_context_t new_context = {684 q, nullptr, &invoke_block, false, false, false, 0};685 cleanup_handler_t new_h = Block_copy(^(int error) {686 {687 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);688 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.689 }690 new_context.orig_context = ^(void) {691 h(error);692 };693 dispatch_callback_wrap(&new_context);694 });695 uptr submit_sync = (uptr)&new_context;696 Release(thr, pc, submit_sync);697 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h);698 Block_release(new_h);699 return new_channel;700}701 702TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path,703 dispatch_io_type_t type, const char *path, int oflag,704 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) {705 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode,706 q, h);707 __block dispatch_io_t new_channel = nullptr;708 __block block_context_t new_context = {709 q, nullptr, &invoke_block, false, false, false, 0};710 cleanup_handler_t new_h = Block_copy(^(int error) {711 {712 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);713 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.714 }715 new_context.orig_context = ^(void) {716 h(error);717 };718 dispatch_callback_wrap(&new_context);719 });720 uptr submit_sync = (uptr)&new_context;721 Release(thr, pc, submit_sync);722 new_channel =723 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h);724 Block_release(new_h);725 return new_channel;726}727 728TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io,729 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q,730 cleanup_handler_t h) {731 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h);732 __block dispatch_io_t new_channel = nullptr;733 __block block_context_t new_context = {734 q, nullptr, &invoke_block, false, false, false, 0};735 cleanup_handler_t new_h = Block_copy(^(int error) {736 {737 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback);738 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close.739 }740 new_context.orig_context = ^(void) {741 h(error);742 };743 dispatch_callback_wrap(&new_context);744 });745 uptr submit_sync = (uptr)&new_context;746 Release(thr, pc, submit_sync);747 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h);748 Block_release(new_h);749 return new_channel;750}751 752TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel,753 dispatch_io_close_flags_t flags) {754 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags);755 Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*].756 return REAL(dispatch_io_close)(channel, flags);757}758 759// Resuming a suspended queue needs to synchronize with all subsequent760// executions of blocks in that queue.761TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) {762 SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o);763 Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync764 // in dispatch_sync_pre_execute765 return REAL(dispatch_resume)(o);766}767 768void InitializeLibdispatchInterceptors() {769 INTERCEPT_FUNCTION(dispatch_async);770 INTERCEPT_FUNCTION(dispatch_async_f);771 INTERCEPT_FUNCTION(dispatch_sync);772 INTERCEPT_FUNCTION(dispatch_sync_f);773 INTERCEPT_FUNCTION(dispatch_barrier_async);774 INTERCEPT_FUNCTION(dispatch_barrier_async_f);775 INTERCEPT_FUNCTION(dispatch_barrier_sync);776 INTERCEPT_FUNCTION(dispatch_barrier_sync_f);777 INTERCEPT_FUNCTION(dispatch_async_and_wait);778 INTERCEPT_FUNCTION(dispatch_async_and_wait_f);779 INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait);780 INTERCEPT_FUNCTION(dispatch_barrier_async_and_wait_f);781 INTERCEPT_FUNCTION(dispatch_after);782 INTERCEPT_FUNCTION(dispatch_after_f);783 INTERCEPT_FUNCTION(dispatch_once);784 INTERCEPT_FUNCTION(dispatch_once_f);785 INTERCEPT_FUNCTION(dispatch_semaphore_signal);786 INTERCEPT_FUNCTION(dispatch_semaphore_wait);787 INTERCEPT_FUNCTION(dispatch_group_wait);788 INTERCEPT_FUNCTION(dispatch_group_leave);789 INTERCEPT_FUNCTION(dispatch_group_async);790 INTERCEPT_FUNCTION(dispatch_group_async_f);791 INTERCEPT_FUNCTION(dispatch_group_notify);792 INTERCEPT_FUNCTION(dispatch_group_notify_f);793 INTERCEPT_FUNCTION(dispatch_source_set_event_handler);794 INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f);795 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler);796 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f);797 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler);798 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f);799 INTERCEPT_FUNCTION(dispatch_apply);800 INTERCEPT_FUNCTION(dispatch_apply_f);801 INTERCEPT_FUNCTION(dispatch_data_create);802 INTERCEPT_FUNCTION(dispatch_read);803 INTERCEPT_FUNCTION(dispatch_write);804 INTERCEPT_FUNCTION(dispatch_io_read);805 INTERCEPT_FUNCTION(dispatch_io_write);806 INTERCEPT_FUNCTION(dispatch_io_barrier);807 INTERCEPT_FUNCTION(dispatch_io_create);808 INTERCEPT_FUNCTION(dispatch_io_create_with_path);809 INTERCEPT_FUNCTION(dispatch_io_create_with_io);810 INTERCEPT_FUNCTION(dispatch_io_close);811 INTERCEPT_FUNCTION(dispatch_resume);812}813 814} // namespace __tsan815