1058 lines · c
1/*2 * kmp_wait_release.h -- Wait/Release implementation3 */4 5//===----------------------------------------------------------------------===//6//7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.8// See https://llvm.org/LICENSE.txt for license information.9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception10//11//===----------------------------------------------------------------------===//12 13#ifndef KMP_WAIT_RELEASE_H14#define KMP_WAIT_RELEASE_H15 16#include "kmp.h"17#include "kmp_itt.h"18#include "kmp_stats.h"19#if OMPT_SUPPORT20#include "ompt-specific.h"21#endif22 23/*!24@defgroup WAIT_RELEASE Wait/Release operations25 26The definitions and functions here implement the lowest level thread27synchronizations of suspending a thread and awaking it. They are used to build28higher level operations such as barriers and fork/join.29*/30 31/*!32@ingroup WAIT_RELEASE33@{34*/35 36struct flag_properties {37 unsigned int type : 16;38 unsigned int reserved : 16;39};40 41template <enum flag_type FlagType> struct flag_traits {};42 43template <> struct flag_traits<flag32> {44 typedef kmp_uint32 flag_t;45 static const flag_type t = flag32;46 static inline flag_t tcr(flag_t f) { return TCR_4(f); }47 static inline flag_t test_then_add4(volatile flag_t *f) {48 return KMP_TEST_THEN_ADD4_32(RCAST(volatile kmp_int32 *, f));49 }50 static inline flag_t test_then_or(volatile flag_t *f, flag_t v) {51 return KMP_TEST_THEN_OR32(f, v);52 }53 static inline flag_t test_then_and(volatile flag_t *f, flag_t v) {54 return KMP_TEST_THEN_AND32(f, v);55 }56};57 58template <> struct flag_traits<atomic_flag64> {59 typedef kmp_uint64 flag_t;60 static const flag_type t = atomic_flag64;61 static inline flag_t tcr(flag_t f) { return TCR_8(f); }62 static inline flag_t test_then_add4(volatile flag_t *f) {63 return KMP_TEST_THEN_ADD4_64(RCAST(volatile kmp_int64 *, f));64 }65 static inline flag_t test_then_or(volatile flag_t *f, flag_t v) {66 return KMP_TEST_THEN_OR64(f, v);67 }68 static inline flag_t test_then_and(volatile flag_t *f, flag_t v) {69 return KMP_TEST_THEN_AND64(f, v);70 }71};72 73template <> struct flag_traits<flag64> {74 typedef kmp_uint64 flag_t;75 static const flag_type t = flag64;76 static inline flag_t tcr(flag_t f) { return TCR_8(f); }77 static inline flag_t test_then_add4(volatile flag_t *f) {78 return KMP_TEST_THEN_ADD4_64(RCAST(volatile kmp_int64 *, f));79 }80 static inline flag_t test_then_or(volatile flag_t *f, flag_t v) {81 return KMP_TEST_THEN_OR64(f, v);82 }83 static inline flag_t test_then_and(volatile flag_t *f, flag_t v) {84 return KMP_TEST_THEN_AND64(f, v);85 }86};87 88template <> struct flag_traits<flag_oncore> {89 typedef kmp_uint64 flag_t;90 static const flag_type t = flag_oncore;91 static inline flag_t tcr(flag_t f) { return TCR_8(f); }92 static inline flag_t test_then_add4(volatile flag_t *f) {93 return KMP_TEST_THEN_ADD4_64(RCAST(volatile kmp_int64 *, f));94 }95 static inline flag_t test_then_or(volatile flag_t *f, flag_t v) {96 return KMP_TEST_THEN_OR64(f, v);97 }98 static inline flag_t test_then_and(volatile flag_t *f, flag_t v) {99 return KMP_TEST_THEN_AND64(f, v);100 }101};102 103/*! Base class for all flags */104template <flag_type FlagType> class kmp_flag {105protected:106 flag_properties t; /**< "Type" of the flag in loc */107 /**< Threads sleeping on this thread. */108 kmp_info_t *waiting_threads[1] = {nullptr};109 kmp_uint32 num_waiting_threads; /**< Num threads sleeping on this thread. */110 std::atomic<bool> *sleepLoc;111 112public:113 typedef flag_traits<FlagType> traits_type;114 kmp_flag() : t({FlagType, 0U}), num_waiting_threads(0), sleepLoc(nullptr) {}115 kmp_flag(int nwaiters)116 : t({FlagType, 0U}), num_waiting_threads(nwaiters), sleepLoc(nullptr) {}117 kmp_flag(std::atomic<bool> *sloc)118 : t({FlagType, 0U}), num_waiting_threads(0), sleepLoc(sloc) {}119 /*! @result the flag_type */120 flag_type get_type() { return (flag_type)(t.type); }121 122 /*! param i in index into waiting_threads123 * @result the thread that is waiting at index i */124 kmp_info_t *get_waiter(kmp_uint32 i) {125 KMP_DEBUG_ASSERT(i < num_waiting_threads);126 return waiting_threads[i];127 }128 /*! @result num_waiting_threads */129 kmp_uint32 get_num_waiters() { return num_waiting_threads; }130 /*! @param thr in the thread which is now waiting131 * Insert a waiting thread at index 0. */132 void set_waiter(kmp_info_t *thr) {133 waiting_threads[0] = thr;134 num_waiting_threads = 1;135 }136 enum barrier_type get_bt() { return bs_last_barrier; }137};138 139/*! Base class for wait/release volatile flag */140template <typename PtrType, flag_type FlagType, bool Sleepable>141class kmp_flag_native : public kmp_flag<FlagType> {142protected:143 volatile PtrType *loc;144 PtrType checker = (PtrType)0; /**< When flag==checker, it has been released */145 typedef flag_traits<FlagType> traits_type;146 147public:148 typedef PtrType flag_t;149 kmp_flag_native(volatile PtrType *p) : kmp_flag<FlagType>(), loc(p) {}150 kmp_flag_native(volatile PtrType *p, kmp_info_t *thr)151 : kmp_flag<FlagType>(1), loc(p) {152 this->waiting_threads[0] = thr;153 }154 kmp_flag_native(volatile PtrType *p, PtrType c)155 : kmp_flag<FlagType>(), loc(p), checker(c) {}156 kmp_flag_native(volatile PtrType *p, PtrType c, std::atomic<bool> *sloc)157 : kmp_flag<FlagType>(sloc), loc(p), checker(c) {}158 virtual ~kmp_flag_native() {}159 void *operator new(size_t size) { return __kmp_allocate(size); }160 void operator delete(void *p) { __kmp_free(p); }161 volatile PtrType *get() { return loc; }162 void *get_void_p() { return RCAST(void *, CCAST(PtrType *, loc)); }163 void set(volatile PtrType *new_loc) { loc = new_loc; }164 PtrType load() { return *loc; }165 void store(PtrType val) { *loc = val; }166 /*! @result true if the flag object has been released. */167 virtual bool done_check() {168 if (Sleepable && !(this->sleepLoc))169 return (traits_type::tcr(*(this->get())) & ~KMP_BARRIER_SLEEP_STATE) ==170 checker;171 else172 return traits_type::tcr(*(this->get())) == checker;173 }174 /*! @param old_loc in old value of flag175 * @result true if the flag's old value indicates it was released. */176 virtual bool done_check_val(PtrType old_loc) { return old_loc == checker; }177 /*! @result true if the flag object is not yet released.178 * Used in __kmp_wait_template like:179 * @code180 * while (flag.notdone_check()) { pause(); }181 * @endcode */182 virtual bool notdone_check() {183 return traits_type::tcr(*(this->get())) != checker;184 }185 /*! @result Actual flag value before release was applied.186 * Trigger all waiting threads to run by modifying flag to release state. */187 void internal_release() {188 (void)traits_type::test_then_add4((volatile PtrType *)this->get());189 }190 /*! @result Actual flag value before sleep bit(s) set.191 * Notes that there is at least one thread sleeping on the flag by setting192 * sleep bit(s). */193 PtrType set_sleeping() {194 if (this->sleepLoc) {195 this->sleepLoc->store(true);196 return *(this->get());197 }198 return traits_type::test_then_or((volatile PtrType *)this->get(),199 KMP_BARRIER_SLEEP_STATE);200 }201 /*! @result Actual flag value before sleep bit(s) cleared.202 * Notes that there are no longer threads sleeping on the flag by clearing203 * sleep bit(s). */204 void unset_sleeping() {205 if (this->sleepLoc) {206 this->sleepLoc->store(false);207 return;208 }209 traits_type::test_then_and((volatile PtrType *)this->get(),210 ~KMP_BARRIER_SLEEP_STATE);211 }212 /*! @param old_loc in old value of flag213 * Test if there are threads sleeping on the flag's old value in old_loc. */214 bool is_sleeping_val(PtrType old_loc) {215 if (this->sleepLoc)216 return this->sleepLoc->load();217 return old_loc & KMP_BARRIER_SLEEP_STATE;218 }219 /*! Test whether there are threads sleeping on the flag. */220 bool is_sleeping() {221 if (this->sleepLoc)222 return this->sleepLoc->load();223 return is_sleeping_val(*(this->get()));224 }225 bool is_any_sleeping() {226 if (this->sleepLoc)227 return this->sleepLoc->load();228 return is_sleeping_val(*(this->get()));229 }230 kmp_uint8 *get_stolen() { return NULL; }231};232 233/*! Base class for wait/release atomic flag */234template <typename PtrType, flag_type FlagType, bool Sleepable>235class kmp_flag_atomic : public kmp_flag<FlagType> {236protected:237 std::atomic<PtrType> *loc; /**< Pointer to flag location to wait on */238 PtrType checker = (PtrType)0; /**< Flag==checker means it has been released */239public:240 typedef flag_traits<FlagType> traits_type;241 typedef PtrType flag_t;242 kmp_flag_atomic(std::atomic<PtrType> *p) : kmp_flag<FlagType>(), loc(p) {}243 kmp_flag_atomic(std::atomic<PtrType> *p, kmp_info_t *thr)244 : kmp_flag<FlagType>(1), loc(p) {245 this->waiting_threads[0] = thr;246 }247 kmp_flag_atomic(std::atomic<PtrType> *p, PtrType c)248 : kmp_flag<FlagType>(), loc(p), checker(c) {}249 kmp_flag_atomic(std::atomic<PtrType> *p, PtrType c, std::atomic<bool> *sloc)250 : kmp_flag<FlagType>(sloc), loc(p), checker(c) {}251 /*! @result the pointer to the actual flag */252 std::atomic<PtrType> *get() { return loc; }253 /*! @result void* pointer to the actual flag */254 void *get_void_p() { return RCAST(void *, loc); }255 /*! @param new_loc in set loc to point at new_loc */256 void set(std::atomic<PtrType> *new_loc) { loc = new_loc; }257 /*! @result flag value */258 PtrType load() { return loc->load(std::memory_order_acquire); }259 /*! @param val the new flag value to be stored */260 void store(PtrType val) { loc->store(val, std::memory_order_release); }261 /*! @result true if the flag object has been released. */262 bool done_check() {263 if (Sleepable && !(this->sleepLoc))264 return (this->load() & ~KMP_BARRIER_SLEEP_STATE) == checker;265 else266 return this->load() == checker;267 }268 /*! @param old_loc in old value of flag269 * @result true if the flag's old value indicates it was released. */270 bool done_check_val(PtrType old_loc) { return old_loc == checker; }271 /*! @result true if the flag object is not yet released.272 * Used in __kmp_wait_template like:273 * @code274 * while (flag.notdone_check()) { pause(); }275 * @endcode */276 bool notdone_check() { return this->load() != checker; }277 /*! @result Actual flag value before release was applied.278 * Trigger all waiting threads to run by modifying flag to release state. */279 void internal_release() { KMP_ATOMIC_ADD(this->get(), 4); }280 /*! @result Actual flag value before sleep bit(s) set.281 * Notes that there is at least one thread sleeping on the flag by setting282 * sleep bit(s). */283 PtrType set_sleeping() {284 if (this->sleepLoc) {285 this->sleepLoc->store(true);286 return *(this->get());287 }288 return KMP_ATOMIC_OR(this->get(), KMP_BARRIER_SLEEP_STATE);289 }290 /*! @result Actual flag value before sleep bit(s) cleared.291 * Notes that there are no longer threads sleeping on the flag by clearing292 * sleep bit(s). */293 void unset_sleeping() {294 if (this->sleepLoc) {295 this->sleepLoc->store(false);296 return;297 }298 KMP_ATOMIC_AND(this->get(), ~KMP_BARRIER_SLEEP_STATE);299 }300 /*! @param old_loc in old value of flag301 * Test whether there are threads sleeping on flag's old value in old_loc. */302 bool is_sleeping_val(PtrType old_loc) {303 if (this->sleepLoc)304 return this->sleepLoc->load();305 return old_loc & KMP_BARRIER_SLEEP_STATE;306 }307 /*! Test whether there are threads sleeping on the flag. */308 bool is_sleeping() {309 if (this->sleepLoc)310 return this->sleepLoc->load();311 return is_sleeping_val(this->load());312 }313 bool is_any_sleeping() {314 if (this->sleepLoc)315 return this->sleepLoc->load();316 return is_sleeping_val(this->load());317 }318 kmp_uint8 *get_stolen() { return NULL; }319};320 321#if OMPT_SUPPORT322OMPT_NOINLINE323static void __ompt_implicit_task_end(kmp_info_t *this_thr,324 ompt_state_t ompt_state,325 ompt_data_t *tId) {326 int ds_tid = this_thr->th.th_info.ds.ds_tid;327 if (ompt_state == ompt_state_wait_barrier_implicit_parallel ||328 ompt_state == ompt_state_wait_barrier_teams) {329 this_thr->th.ompt_thread_info.state = ompt_state_overhead;330#if OMPT_OPTIONAL331 void *codeptr = NULL;332 ompt_sync_region_t sync_kind = ompt_sync_region_barrier_implicit_parallel;333 if (this_thr->th.ompt_thread_info.parallel_flags & ompt_parallel_league)334 sync_kind = ompt_sync_region_barrier_teams;335 if (ompt_enabled.ompt_callback_sync_region_wait) {336 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(337 sync_kind, ompt_scope_end, NULL, tId, codeptr);338 }339 if (ompt_enabled.ompt_callback_sync_region) {340 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(341 sync_kind, ompt_scope_end, NULL, tId, codeptr);342 }343#endif344 if (!KMP_MASTER_TID(ds_tid)) {345 if (ompt_enabled.ompt_callback_implicit_task) {346 int flags = this_thr->th.ompt_thread_info.parallel_flags;347 flags = (flags & ompt_parallel_league) ? ompt_task_initial348 : ompt_task_implicit;349 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(350 ompt_scope_end, NULL, tId, 0, ds_tid, flags);351 }352 // return to idle state353 this_thr->th.ompt_thread_info.state = ompt_state_idle;354 } else {355 this_thr->th.ompt_thread_info.state = ompt_state_overhead;356 }357 }358}359#endif360 361/* Spin wait loop that first does pause/yield, then sleep. A thread that calls362 __kmp_wait_* must make certain that another thread calls __kmp_release363 to wake it back up to prevent deadlocks!364 365 NOTE: We may not belong to a team at this point. */366template <class C, bool final_spin, bool Cancellable = false,367 bool Sleepable = true>368static inline bool369__kmp_wait_template(kmp_info_t *this_thr,370 C *flag USE_ITT_BUILD_ARG(void *itt_sync_obj)) {371#if USE_ITT_BUILD && USE_ITT_NOTIFY372 volatile void *spin = flag->get();373#endif374 kmp_uint32 spins;375 int th_gtid;376 int tasks_completed = FALSE;377#if !KMP_USE_MONITOR378 kmp_uint64 poll_count;379 kmp_uint64 hibernate_goal;380#else381 kmp_uint32 hibernate;382#endif383 kmp_uint64 time;384 385 KMP_FSYNC_SPIN_INIT(spin, NULL);386 if (flag->done_check()) {387 KMP_FSYNC_SPIN_ACQUIRED(CCAST(void *, spin));388 return false;389 }390 th_gtid = this_thr->th.th_info.ds.ds_gtid;391 if (Cancellable) {392 kmp_team_t *team = this_thr->th.th_team;393 if (team && team->t.t_cancel_request == cancel_parallel)394 return true;395 }396#if KMP_OS_UNIX397 if (final_spin)398 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true);399#endif400 KA_TRACE(20,401 ("__kmp_wait_sleep: T#%d waiting for flag(%p)\n", th_gtid, flag));402#if KMP_STATS_ENABLED403 stats_state_e thread_state = KMP_GET_THREAD_STATE();404#endif405 406/* OMPT Behavior:407THIS function is called from408 __kmp_barrier (2 times) (implicit or explicit barrier in parallel regions)409 these have join / fork behavior410 411 In these cases, we don't change the state or trigger events in THIS412function.413 Events are triggered in the calling code (__kmp_barrier):414 415 state := ompt_state_overhead416 barrier-begin417 barrier-wait-begin418 state := ompt_state_wait_barrier419 call join-barrier-implementation (finally arrive here)420 {}421 call fork-barrier-implementation (finally arrive here)422 {}423 state := ompt_state_overhead424 barrier-wait-end425 barrier-end426 state := ompt_state_work_parallel427 428 429 __kmp_fork_barrier (after thread creation, before executing implicit task)430 call fork-barrier-implementation (finally arrive here)431 {} // worker arrive here with state = ompt_state_idle432 433 434 __kmp_join_barrier (implicit barrier at end of parallel region)435 state := ompt_state_barrier_implicit436 barrier-begin437 barrier-wait-begin438 call join-barrier-implementation (finally arrive here439final_spin=FALSE)440 {441 }442 __kmp_fork_barrier (implicit barrier at end of parallel region)443 call fork-barrier-implementation (finally arrive here final_spin=TRUE)444 445 Worker after task-team is finished:446 barrier-wait-end447 barrier-end448 implicit-task-end449 idle-begin450 state := ompt_state_idle451 452 Before leaving, if state = ompt_state_idle453 idle-end454 state := ompt_state_overhead455*/456#if OMPT_SUPPORT457 ompt_state_t ompt_entry_state;458 ompt_data_t *tId;459 if (ompt_enabled.enabled) {460 ompt_entry_state = this_thr->th.ompt_thread_info.state;461 if (!final_spin ||462 (ompt_entry_state != ompt_state_wait_barrier_implicit_parallel &&463 ompt_entry_state != ompt_state_wait_barrier_teams) ||464 KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid)) {465 ompt_lw_taskteam_t *team = NULL;466 if (this_thr->th.th_team)467 team = this_thr->th.th_team->t.ompt_serialized_team_info;468 if (team) {469 tId = &(team->ompt_task_info.task_data);470 } else {471 tId = OMPT_CUR_TASK_DATA(this_thr);472 }473 } else {474 tId = &(this_thr->th.ompt_thread_info.task_data);475 }476 if (final_spin && (__kmp_tasking_mode == tskm_immediate_exec ||477 this_thr->th.th_task_team == NULL)) {478 // implicit task is done. Either no taskqueue, or task-team finished479 __ompt_implicit_task_end(this_thr, ompt_entry_state, tId);480 }481 }482#endif483 484 KMP_INIT_YIELD(spins); // Setup for waiting485 KMP_INIT_BACKOFF(time);486 487 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ||488 __kmp_pause_status == kmp_soft_paused) {489#if KMP_USE_MONITOR490// The worker threads cannot rely on the team struct existing at this point.491// Use the bt values cached in the thread struct instead.492#ifdef KMP_ADJUST_BLOCKTIME493 if (__kmp_pause_status == kmp_soft_paused ||494 (__kmp_zero_bt && !this_thr->th.th_team_bt_set))495 // Force immediate suspend if not set by user and more threads than496 // available procs497 hibernate = 0;498 else499 hibernate = this_thr->th.th_team_bt_intervals;500#else501 hibernate = this_thr->th.th_team_bt_intervals;502#endif /* KMP_ADJUST_BLOCKTIME */503 504 /* If the blocktime is nonzero, we want to make sure that we spin wait for505 the entirety of the specified #intervals, plus up to one interval more.506 This increment make certain that this thread doesn't go to sleep too507 soon. */508 if (hibernate != 0)509 hibernate++;510 511 // Add in the current time value.512 hibernate += TCR_4(__kmp_global.g.g_time.dt.t_value);513 KF_TRACE(20, ("__kmp_wait_sleep: T#%d now=%d, hibernate=%d, intervals=%d\n",514 th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate,515 hibernate - __kmp_global.g.g_time.dt.t_value));516#else517 if (__kmp_pause_status == kmp_soft_paused) {518 // Force immediate suspend519 hibernate_goal = KMP_NOW();520 } else521 hibernate_goal = KMP_NOW() + this_thr->th.th_team_bt_intervals;522 poll_count = 0;523 (void)poll_count;524#endif // KMP_USE_MONITOR525 }526 527 KMP_MB();528 529 // Main wait spin loop530 while (flag->notdone_check()) {531 kmp_task_team_t *task_team = NULL;532 if (__kmp_tasking_mode != tskm_immediate_exec) {533 task_team = this_thr->th.th_task_team;534 /* If the thread's task team pointer is NULL, it means one of 3 things:535 1) A newly-created thread is first being released by536 __kmp_fork_barrier(), and its task team has not been set up yet.537 2) All tasks have been executed to completion.538 3) Tasking is off for this region. This could be because we are in a539 serialized region (perhaps the outer one), or else tasking was manually540 disabled (KMP_TASKING=0). */541 if (task_team != NULL) {542 if (TCR_SYNC_4(task_team->tt.tt_active)) {543 if (KMP_TASKING_ENABLED(task_team)) {544 flag->execute_tasks(545 this_thr, th_gtid, final_spin,546 &tasks_completed USE_ITT_BUILD_ARG(itt_sync_obj), 0);547 } else548 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;549 } else {550 KMP_DEBUG_ASSERT(!KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid));551#if OMPT_SUPPORT552 // task-team is done now, other cases should be catched above553 if (final_spin && ompt_enabled.enabled)554 __ompt_implicit_task_end(this_thr, ompt_entry_state, tId);555#endif556 this_thr->th.th_task_team = NULL;557 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;558 }559 } else {560 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;561 } // if562 } // if563 564 KMP_FSYNC_SPIN_PREPARE(CCAST(void *, spin));565 if (TCR_4(__kmp_global.g.g_done)) {566 if (__kmp_global.g.g_abort)567 __kmp_abort_thread();568 break;569 }570 571 // If we are oversubscribed, or have waited a bit (and572 // KMP_LIBRARY=throughput), then yield573 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time);574 575#if KMP_STATS_ENABLED576 // Check if thread has been signalled to idle state577 // This indicates that the logical "join-barrier" has finished578 if (this_thr->th.th_stats->isIdle() &&579 KMP_GET_THREAD_STATE() == FORK_JOIN_BARRIER) {580 KMP_SET_THREAD_STATE(IDLE);581 KMP_PUSH_PARTITIONED_TIMER(OMP_idle);582 }583#endif584 // Check if the barrier surrounding this wait loop has been cancelled585 if (Cancellable) {586 kmp_team_t *team = this_thr->th.th_team;587 if (team && team->t.t_cancel_request == cancel_parallel)588 break;589 }590 591 // For hidden helper thread, if task_team is nullptr, it means the main592 // thread has not released the barrier. We cannot wait here because once the593 // main thread releases all children barriers, all hidden helper threads are594 // still sleeping. This leads to a problem that following configuration,595 // such as task team sync, will not be performed such that this thread does596 // not have task team. Usually it is not bad. However, a corner case is,597 // when the first task encountered is an untied task, the check in598 // __kmp_task_alloc will crash because it uses the task team pointer without599 // checking whether it is nullptr. It is probably under some kind of600 // assumption.601 if (task_team && KMP_HIDDEN_HELPER_WORKER_THREAD(th_gtid) &&602 !TCR_4(__kmp_hidden_helper_team_done)) {603 // If there is still hidden helper tasks to be executed, the hidden helper604 // thread will not enter a waiting status.605 if (KMP_ATOMIC_LD_ACQ(&__kmp_unexecuted_hidden_helper_tasks) == 0) {606 __kmp_hidden_helper_worker_thread_wait();607 }608 continue;609 }610 611 // Don't suspend if KMP_BLOCKTIME is set to "infinite"612 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&613 __kmp_pause_status != kmp_soft_paused)614 continue;615 616 // Don't suspend if there is a likelihood of new tasks being spawned.617 if (task_team != NULL && TCR_4(task_team->tt.tt_found_tasks) &&618 !__kmp_wpolicy_passive)619 continue;620 621#if KMP_USE_MONITOR622 // If we have waited a bit more, fall asleep623 if (TCR_4(__kmp_global.g.g_time.dt.t_value) < hibernate)624 continue;625#else626 if (KMP_BLOCKING(hibernate_goal, poll_count++))627 continue;628#endif629 // Don't suspend if wait loop designated non-sleepable630 // in template parameters631 if (!Sleepable)632 continue;633 634#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT635 if (__kmp_mwait_enabled || __kmp_umwait_enabled) {636 KF_TRACE(50, ("__kmp_wait_sleep: T#%d using monitor/mwait\n", th_gtid));637 flag->mwait(th_gtid);638 } else {639#endif640 KF_TRACE(50, ("__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid));641#if KMP_OS_UNIX642 if (final_spin)643 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false);644#endif645 flag->suspend(th_gtid);646#if KMP_OS_UNIX647 if (final_spin)648 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, true);649#endif650#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT651 }652#endif653 654 if (TCR_4(__kmp_global.g.g_done)) {655 if (__kmp_global.g.g_abort)656 __kmp_abort_thread();657 break;658 } else if (__kmp_tasking_mode != tskm_immediate_exec &&659 this_thr->th.th_reap_state == KMP_SAFE_TO_REAP) {660 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;661 }662 // TODO: If thread is done with work and times out, disband/free663 }664 665#if OMPT_SUPPORT666 ompt_state_t ompt_exit_state = this_thr->th.ompt_thread_info.state;667 if (ompt_enabled.enabled && ompt_exit_state != ompt_state_undefined) {668#if OMPT_OPTIONAL669 if (final_spin) {670 __ompt_implicit_task_end(this_thr, ompt_exit_state, tId);671 ompt_exit_state = this_thr->th.ompt_thread_info.state;672 }673#endif674 if (ompt_exit_state == ompt_state_idle) {675 this_thr->th.ompt_thread_info.state = ompt_state_overhead;676 }677 }678#endif679#if KMP_STATS_ENABLED680 // If we were put into idle state, pop that off the state stack681 if (KMP_GET_THREAD_STATE() == IDLE) {682 KMP_POP_PARTITIONED_TIMER();683 KMP_SET_THREAD_STATE(thread_state);684 this_thr->th.th_stats->resetIdleFlag();685 }686#endif687 688#if KMP_OS_UNIX689 if (final_spin)690 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking, false);691#endif692 KMP_FSYNC_SPIN_ACQUIRED(CCAST(void *, spin));693 if (Cancellable) {694 kmp_team_t *team = this_thr->th.th_team;695 if (team && team->t.t_cancel_request == cancel_parallel) {696 if (tasks_completed) {697 // undo the previous decrement of unfinished_threads so that the698 // thread can decrement at the join barrier with no problem699 kmp_task_team_t *task_team = this_thr->th.th_task_team;700 std::atomic<kmp_int32> *unfinished_threads =701 &(task_team->tt.tt_unfinished_threads);702 KMP_ATOMIC_INC(unfinished_threads);703 }704 return true;705 }706 }707 return false;708}709 710#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT711// Set up a monitor on the flag variable causing the calling thread to wait in712// a less active state until the flag variable is modified.713template <class C>714static inline void __kmp_mwait_template(int th_gtid, C *flag) {715 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_mwait);716 kmp_info_t *th = __kmp_threads[th_gtid];717 718 KF_TRACE(30, ("__kmp_mwait_template: T#%d enter for flag = %p\n", th_gtid,719 flag->get()));720 721 // User-level mwait is available722 KMP_DEBUG_ASSERT(__kmp_mwait_enabled || __kmp_umwait_enabled);723 724 __kmp_suspend_initialize_thread(th);725 __kmp_lock_suspend_mx(th);726 727 volatile void *spin = flag->get();728 void *cacheline = (void *)(kmp_uintptr_t(spin) & ~(CACHE_LINE - 1));729 730 if (!flag->done_check()) {731 // Mark thread as no longer active732 th->th.th_active = FALSE;733 if (th->th.th_active_in_pool) {734 th->th.th_active_in_pool = FALSE;735 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);736 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);737 }738 flag->set_sleeping();739 KF_TRACE(50, ("__kmp_mwait_template: T#%d calling monitor\n", th_gtid));740#if KMP_HAVE_UMWAIT741 if (__kmp_umwait_enabled) {742 __kmp_umonitor(cacheline);743 }744#elif KMP_HAVE_MWAIT745 if (__kmp_mwait_enabled) {746 __kmp_mm_monitor(cacheline, 0, 0);747 }748#endif749 // To avoid a race, check flag between 'monitor' and 'mwait'. A write to750 // the address could happen after the last time we checked and before751 // monitoring started, in which case monitor can't detect the change.752 if (flag->done_check())753 flag->unset_sleeping();754 else {755 // if flag changes here, wake-up happens immediately756 TCW_PTR(th->th.th_sleep_loc, (void *)flag);757 th->th.th_sleep_loc_type = flag->get_type();758 __kmp_unlock_suspend_mx(th);759 KF_TRACE(50, ("__kmp_mwait_template: T#%d calling mwait\n", th_gtid));760#if KMP_HAVE_UMWAIT761 if (__kmp_umwait_enabled) {762 __kmp_umwait(1, 100); // to do: enable ctrl via hints, backoff counter763 }764#elif KMP_HAVE_MWAIT765 if (__kmp_mwait_enabled) {766 __kmp_mm_mwait(0, __kmp_mwait_hints);767 }768#endif769 KF_TRACE(50, ("__kmp_mwait_template: T#%d mwait done\n", th_gtid));770 __kmp_lock_suspend_mx(th);771 // Clean up sleep info; doesn't matter how/why this thread stopped waiting772 if (flag->is_sleeping())773 flag->unset_sleeping();774 TCW_PTR(th->th.th_sleep_loc, NULL);775 th->th.th_sleep_loc_type = flag_unset;776 }777 // Mark thread as active again778 th->th.th_active = TRUE;779 if (TCR_4(th->th.th_in_pool)) {780 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);781 th->th.th_active_in_pool = TRUE;782 }783 } // Drop out to main wait loop to check flag, handle tasks, etc.784 __kmp_unlock_suspend_mx(th);785 KF_TRACE(30, ("__kmp_mwait_template: T#%d exit\n", th_gtid));786}787#endif // KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT788 789/* Release any threads specified as waiting on the flag by releasing the flag790 and resume the waiting thread if indicated by the sleep bit(s). A thread that791 calls __kmp_wait_template must call this function to wake up the potentially792 sleeping thread and prevent deadlocks! */793template <class C> static inline void __kmp_release_template(C *flag) {794#ifdef KMP_DEBUG795 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;796#endif797 KF_TRACE(20, ("__kmp_release: T#%d releasing flag(%x)\n", gtid, flag->get()));798 KMP_DEBUG_ASSERT(flag->get());799 KMP_FSYNC_RELEASING(flag->get_void_p());800 801 flag->internal_release();802 803 KF_TRACE(100, ("__kmp_release: T#%d set new spin=%d\n", gtid, flag->get(),804 flag->load()));805 806 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {807 // Only need to check sleep stuff if infinite block time not set.808 // Are *any* threads waiting on flag sleeping?809 if (flag->is_any_sleeping()) {810 for (unsigned int i = 0; i < flag->get_num_waiters(); ++i) {811 // if sleeping waiter exists at i, sets current_waiter to i inside flag812 kmp_info_t *waiter = flag->get_waiter(i);813 if (waiter) {814 int wait_gtid = waiter->th.th_info.ds.ds_gtid;815 // Wake up thread if needed816 KF_TRACE(50, ("__kmp_release: T#%d waking up thread T#%d since sleep "817 "flag(%p) set\n",818 gtid, wait_gtid, flag->get()));819 flag->resume(wait_gtid); // unsets flag's current_waiter when done820 }821 }822 }823 }824}825 826template <bool Cancellable, bool Sleepable>827class kmp_flag_32 : public kmp_flag_atomic<kmp_uint32, flag32, Sleepable> {828public:829 kmp_flag_32(std::atomic<kmp_uint32> *p)830 : kmp_flag_atomic<kmp_uint32, flag32, Sleepable>(p) {}831 kmp_flag_32(std::atomic<kmp_uint32> *p, kmp_info_t *thr)832 : kmp_flag_atomic<kmp_uint32, flag32, Sleepable>(p, thr) {}833 kmp_flag_32(std::atomic<kmp_uint32> *p, kmp_uint32 c)834 : kmp_flag_atomic<kmp_uint32, flag32, Sleepable>(p, c) {}835 void suspend(int th_gtid) { __kmp_suspend_32(th_gtid, this); }836#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT837 void mwait(int th_gtid) { __kmp_mwait_32(th_gtid, this); }838#endif839 void resume(int th_gtid) { __kmp_resume_32(th_gtid, this); }840 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,841 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),842 kmp_int32 is_constrained) {843 return __kmp_execute_tasks_32(844 this_thr, gtid, this, final_spin,845 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);846 }847 bool wait(kmp_info_t *this_thr,848 int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) {849 if (final_spin)850 return __kmp_wait_template<kmp_flag_32, TRUE, Cancellable, Sleepable>(851 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));852 else853 return __kmp_wait_template<kmp_flag_32, FALSE, Cancellable, Sleepable>(854 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));855 }856 void release() { __kmp_release_template(this); }857 flag_type get_ptr_type() { return flag32; }858};859 860template <bool Cancellable, bool Sleepable>861class kmp_flag_64 : public kmp_flag_native<kmp_uint64, flag64, Sleepable> {862public:863 kmp_flag_64(volatile kmp_uint64 *p)864 : kmp_flag_native<kmp_uint64, flag64, Sleepable>(p) {}865 kmp_flag_64(volatile kmp_uint64 *p, kmp_info_t *thr)866 : kmp_flag_native<kmp_uint64, flag64, Sleepable>(p, thr) {}867 kmp_flag_64(volatile kmp_uint64 *p, kmp_uint64 c)868 : kmp_flag_native<kmp_uint64, flag64, Sleepable>(p, c) {}869 kmp_flag_64(volatile kmp_uint64 *p, kmp_uint64 c, std::atomic<bool> *loc)870 : kmp_flag_native<kmp_uint64, flag64, Sleepable>(p, c, loc) {}871 void suspend(int th_gtid) { __kmp_suspend_64(th_gtid, this); }872#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT873 void mwait(int th_gtid) { __kmp_mwait_64(th_gtid, this); }874#endif875 void resume(int th_gtid) { __kmp_resume_64(th_gtid, this); }876 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,877 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),878 kmp_int32 is_constrained) {879 return __kmp_execute_tasks_64(880 this_thr, gtid, this, final_spin,881 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);882 }883 bool wait(kmp_info_t *this_thr,884 int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) {885 if (final_spin)886 return __kmp_wait_template<kmp_flag_64, TRUE, Cancellable, Sleepable>(887 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));888 else889 return __kmp_wait_template<kmp_flag_64, FALSE, Cancellable, Sleepable>(890 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));891 }892 void release() { __kmp_release_template(this); }893 flag_type get_ptr_type() { return flag64; }894};895 896template <bool Cancellable, bool Sleepable>897class kmp_atomic_flag_64898 : public kmp_flag_atomic<kmp_uint64, atomic_flag64, Sleepable> {899public:900 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p)901 : kmp_flag_atomic<kmp_uint64, atomic_flag64, Sleepable>(p) {}902 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p, kmp_info_t *thr)903 : kmp_flag_atomic<kmp_uint64, atomic_flag64, Sleepable>(p, thr) {}904 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p, kmp_uint64 c)905 : kmp_flag_atomic<kmp_uint64, atomic_flag64, Sleepable>(p, c) {}906 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p, kmp_uint64 c,907 std::atomic<bool> *loc)908 : kmp_flag_atomic<kmp_uint64, atomic_flag64, Sleepable>(p, c, loc) {}909 void suspend(int th_gtid) { __kmp_atomic_suspend_64(th_gtid, this); }910 void mwait(int th_gtid) { __kmp_atomic_mwait_64(th_gtid, this); }911 void resume(int th_gtid) { __kmp_atomic_resume_64(th_gtid, this); }912 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,913 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),914 kmp_int32 is_constrained) {915 return __kmp_atomic_execute_tasks_64(916 this_thr, gtid, this, final_spin,917 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);918 }919 bool wait(kmp_info_t *this_thr,920 int final_spin USE_ITT_BUILD_ARG(void *itt_sync_obj)) {921 if (final_spin)922 return __kmp_wait_template<kmp_atomic_flag_64, TRUE, Cancellable,923 Sleepable>(924 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));925 else926 return __kmp_wait_template<kmp_atomic_flag_64, FALSE, Cancellable,927 Sleepable>(928 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));929 }930 void release() { __kmp_release_template(this); }931 flag_type get_ptr_type() { return atomic_flag64; }932};933 934// Hierarchical 64-bit on-core barrier instantiation935class kmp_flag_oncore : public kmp_flag_native<kmp_uint64, flag_oncore, false> {936 kmp_uint32 offset; /**< Portion of flag of interest for an operation. */937 bool flag_switch; /**< Indicates a switch in flag location. */938 enum barrier_type bt; /**< Barrier type. */939 /**< Thread to redirect to different flag location. */940 kmp_info_t *this_thr = nullptr;941#if USE_ITT_BUILD942 void *itt_sync_obj; /**< ITT object to pass to new flag location. */943#endif944 unsigned char &byteref(volatile kmp_uint64 *loc, size_t offset) {945 return (RCAST(unsigned char *, CCAST(kmp_uint64 *, loc)))[offset];946 }947 948public:949 kmp_flag_oncore(volatile kmp_uint64 *p)950 : kmp_flag_native<kmp_uint64, flag_oncore, false>(p), flag_switch(false) {951 }952 kmp_flag_oncore(volatile kmp_uint64 *p, kmp_uint32 idx)953 : kmp_flag_native<kmp_uint64, flag_oncore, false>(p), offset(idx),954 flag_switch(false),955 bt(bs_last_barrier) USE_ITT_BUILD_ARG(itt_sync_obj(nullptr)) {}956 kmp_flag_oncore(volatile kmp_uint64 *p, kmp_uint64 c, kmp_uint32 idx,957 enum barrier_type bar_t,958 kmp_info_t *thr USE_ITT_BUILD_ARG(void *itt))959 : kmp_flag_native<kmp_uint64, flag_oncore, false>(p, c), offset(idx),960 flag_switch(false), bt(bar_t),961 this_thr(thr) USE_ITT_BUILD_ARG(itt_sync_obj(itt)) {}962 virtual ~kmp_flag_oncore() override {}963 void *operator new(size_t size) { return __kmp_allocate(size); }964 void operator delete(void *p) { __kmp_free(p); }965 bool done_check_val(kmp_uint64 old_loc) override {966 return byteref(&old_loc, offset) == checker;967 }968 bool done_check() override { return done_check_val(*get()); }969 bool notdone_check() override {970 // Calculate flag_switch971 if (this_thr->th.th_bar[bt].bb.wait_flag == KMP_BARRIER_SWITCH_TO_OWN_FLAG)972 flag_switch = true;973 if (byteref(get(), offset) != 1 && !flag_switch)974 return true;975 else if (flag_switch) {976 this_thr->th.th_bar[bt].bb.wait_flag = KMP_BARRIER_SWITCHING;977 kmp_flag_64<> flag(&this_thr->th.th_bar[bt].bb.b_go,978 (kmp_uint64)KMP_BARRIER_STATE_BUMP);979 __kmp_wait_64(this_thr, &flag, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));980 }981 return false;982 }983 void internal_release() {984 // Other threads can write their own bytes simultaneously.985 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {986 byteref(get(), offset) = 1;987 } else {988 kmp_uint64 mask = 0;989 byteref(&mask, offset) = 1;990 KMP_TEST_THEN_OR64(get(), mask);991 }992 }993 void wait(kmp_info_t *this_thr, int final_spin) {994 if (final_spin)995 __kmp_wait_template<kmp_flag_oncore, TRUE>(996 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));997 else998 __kmp_wait_template<kmp_flag_oncore, FALSE>(999 this_thr, this USE_ITT_BUILD_ARG(itt_sync_obj));1000 }1001 void release() { __kmp_release_template(this); }1002 void suspend(int th_gtid) { __kmp_suspend_oncore(th_gtid, this); }1003#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT1004 void mwait(int th_gtid) { __kmp_mwait_oncore(th_gtid, this); }1005#endif1006 void resume(int th_gtid) { __kmp_resume_oncore(th_gtid, this); }1007 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid, int final_spin,1008 int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),1009 kmp_int32 is_constrained) {1010#if OMPD_SUPPORT1011 int ret = __kmp_execute_tasks_oncore(1012 this_thr, gtid, this, final_spin,1013 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);1014 if (ompd_state & OMPD_ENABLE_BP)1015 ompd_bp_task_end();1016 return ret;1017#else1018 return __kmp_execute_tasks_oncore(1019 this_thr, gtid, this, final_spin,1020 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);1021#endif1022 }1023 enum barrier_type get_bt() { return bt; }1024 flag_type get_ptr_type() { return flag_oncore; }1025};1026 1027static inline void __kmp_null_resume_wrapper(kmp_info_t *thr) {1028 int gtid = __kmp_gtid_from_thread(thr);1029 void *flag = CCAST(void *, thr->th.th_sleep_loc);1030 flag_type type = thr->th.th_sleep_loc_type;1031 if (!flag)1032 return;1033 // Attempt to wake up a thread: examine its type and call appropriate template1034 switch (type) {1035 case flag32:1036 __kmp_resume_32(gtid, RCAST(kmp_flag_32<> *, flag));1037 break;1038 case flag64:1039 __kmp_resume_64(gtid, RCAST(kmp_flag_64<> *, flag));1040 break;1041 case atomic_flag64:1042 __kmp_atomic_resume_64(gtid, RCAST(kmp_atomic_flag_64<> *, flag));1043 break;1044 case flag_oncore:1045 __kmp_resume_oncore(gtid, RCAST(kmp_flag_oncore *, flag));1046 break;1047 case flag_unset:1048 KF_TRACE(100, ("__kmp_null_resume_wrapper: flag type %d is unset\n", type));1049 break;1050 }1051}1052 1053/*!1054@}1055*/1056 1057#endif // KMP_WAIT_RELEASE_H1058