4894 lines · c
1/*! \file */2/*3 * kmp.h -- KPTS runtime header file.4 */5 6//===----------------------------------------------------------------------===//7//8// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.9// See https://llvm.org/LICENSE.txt for license information.10// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception11//12//===----------------------------------------------------------------------===//13 14#ifndef KMP_H15#define KMP_H16 17#include "kmp_config.h"18 19/* #define BUILD_PARALLEL_ORDERED 1 */20 21/* This fix replaces gettimeofday with clock_gettime for better scalability on22 the Altix. Requires user code to be linked with -lrt. */23//#define FIX_SGI_CLOCK24 25/* Defines for OpenMP 3.0 tasking and auto scheduling */26 27#ifndef KMP_STATIC_STEAL_ENABLED28#define KMP_STATIC_STEAL_ENABLED 129#endif30#define KMP_WEIGHTED_ITERATIONS_SUPPORTED \31 (KMP_AFFINITY_SUPPORTED && KMP_STATIC_STEAL_ENABLED && \32 (KMP_ARCH_X86 || KMP_ARCH_X86_64))33 34#define TASK_CURRENT_NOT_QUEUED 035#define TASK_CURRENT_QUEUED 136 37#define TASK_NOT_PUSHED 138#define TASK_SUCCESSFULLY_PUSHED 039#define TASK_TIED 140#define TASK_UNTIED 041#define TASK_EXPLICIT 142#define TASK_IMPLICIT 043#define TASK_PROXY 144#define TASK_FULL 045#define TASK_DETACHABLE 146#define TASK_UNDETACHABLE 047 48#define KMP_CANCEL_THREADS49#define KMP_THREAD_ATTR50 51// Android does not have pthread_cancel. Undefine KMP_CANCEL_THREADS if being52// built on Android53#if defined(__ANDROID__)54#undef KMP_CANCEL_THREADS55#endif56 57// Some WASI targets (e.g., wasm32-wasi-threads) do not support thread58// cancellation.59#if KMP_OS_WASI60#undef KMP_CANCEL_THREADS61#endif62 63#if !KMP_OS_WASI64#include <signal.h>65#endif66#include <stdarg.h>67#include <stddef.h>68#include <stdio.h>69#include <stdlib.h>70#include <string.h>71#include <limits>72#include <type_traits>73/* include <ctype.h> don't use; problems with /MD on Windows* OS NT due to bad74 Microsoft library. Some macros provided below to replace these functions */75#ifndef __ABSOFT_WIN76#include <sys/types.h>77#endif78#include <limits.h>79#include <time.h>80 81#include <errno.h>82 83#include "kmp_os.h"84 85#include "kmp_safe_c_api.h"86 87#if KMP_STATS_ENABLED88class kmp_stats_list;89#endif90 91#if KMP_USE_HIER_SCHED92// Only include hierarchical scheduling if affinity is supported93#undef KMP_USE_HIER_SCHED94#define KMP_USE_HIER_SCHED KMP_AFFINITY_SUPPORTED95#endif96 97// OMPD_SKIP_HWLOC used in libompd/omp-icv.cpp to avoid OMPD depending on hwloc98#if KMP_USE_HWLOC && KMP_AFFINITY_SUPPORTED && !defined(OMPD_SKIP_HWLOC)99#include "hwloc.h"100#define KMP_HWLOC_ENABLED 1101#ifndef HWLOC_OBJ_NUMANODE102#define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE103#endif104#ifndef HWLOC_OBJ_PACKAGE105#define HWLOC_OBJ_PACKAGE HWLOC_OBJ_SOCKET106#endif107#else108#define KMP_HWLOC_ENABLED 0109#endif110 111#if KMP_ARCH_X86 || KMP_ARCH_X86_64112#include <xmmintrin.h>113#endif114 115// The below has to be defined before including "kmp_barrier.h".116#define KMP_INTERNAL_MALLOC(sz) malloc(sz)117#define KMP_INTERNAL_FREE(p) free(p)118#define KMP_INTERNAL_REALLOC(p, sz) realloc((p), (sz))119#define KMP_INTERNAL_CALLOC(n, sz) calloc((n), (sz))120 121#include "kmp_debug.h"122#include "kmp_lock.h"123#include "kmp_version.h"124#include "kmp_barrier.h"125#if USE_DEBUGGER126#include "kmp_debugger.h"127#endif128#include "kmp_i18n.h"129 130#define KMP_HANDLE_SIGNALS ((KMP_OS_UNIX && !KMP_OS_WASI) || KMP_OS_WINDOWS)131 132#include "kmp_wrapper_malloc.h"133#if KMP_OS_UNIX134#include <unistd.h>135#if !defined NSIG && defined _NSIG136#define NSIG _NSIG137#endif138#endif139 140#if KMP_OS_LINUX141#pragma weak clock_gettime142#endif143 144#if OMPT_SUPPORT145#include "ompt-internal.h"146#endif147 148#if OMPD_SUPPORT149#include "ompd-specific.h"150#endif151 152#ifndef UNLIKELY153#define UNLIKELY(x) (x)154#endif155 156// Affinity format function157#include "kmp_str.h"158 159// 0 - no fast memory allocation, alignment: 8-byte on x86, 16-byte on x64.160// 3 - fast allocation using sync, non-sync free lists of any size, non-self161// free lists of limited size.162#ifndef USE_FAST_MEMORY163#define USE_FAST_MEMORY 3164#endif165 166// Assume using BGET compare_exchange instruction instead of lock by default.167#ifndef USE_CMP_XCHG_FOR_BGET168#define USE_CMP_XCHG_FOR_BGET 1169#endif170 171// Test to see if queuing lock is better than bootstrap lock for bget172// #ifndef USE_QUEUING_LOCK_FOR_BGET173// #define USE_QUEUING_LOCK_FOR_BGET174// #endif175 176#define KMP_NSEC_PER_SEC 1000000000L177#define KMP_USEC_PER_SEC 1000000L178#define KMP_NSEC_PER_USEC 1000L179 180/*!181@ingroup BASIC_TYPES182@{183*/184 185/*!186Values for bit flags used in the ident_t to describe the fields.187*/188enum {189 /*! Use trampoline for internal microtasks */190 KMP_IDENT_IMB = 0x01,191 /*! Use c-style ident structure */192 KMP_IDENT_KMPC = 0x02,193 /* 0x04 is no longer used */194 /*! Entry point generated by auto-parallelization */195 KMP_IDENT_AUTOPAR = 0x08,196 /*! Compiler generates atomic reduction option for kmpc_reduce* */197 KMP_IDENT_ATOMIC_REDUCE = 0x10,198 /*! To mark a 'barrier' directive in user code */199 KMP_IDENT_BARRIER_EXPL = 0x20,200 /*! To Mark implicit barriers. */201 KMP_IDENT_BARRIER_IMPL = 0x0040,202 KMP_IDENT_BARRIER_IMPL_MASK = 0x01C0,203 KMP_IDENT_BARRIER_IMPL_FOR = 0x0040,204 KMP_IDENT_BARRIER_IMPL_SECTIONS = 0x00C0,205 206 KMP_IDENT_BARRIER_IMPL_SINGLE = 0x0140,207 KMP_IDENT_BARRIER_IMPL_WORKSHARE = 0x01C0,208 209 /*! To mark a static loop in OMPT callbacks */210 KMP_IDENT_WORK_LOOP = 0x200,211 /*! To mark a sections directive in OMPT callbacks */212 KMP_IDENT_WORK_SECTIONS = 0x400,213 /*! To mark a distribute construct in OMPT callbacks */214 KMP_IDENT_WORK_DISTRIBUTE = 0x800,215 /*! Atomic hint; bottom four bits as omp_sync_hint_t. Top four reserved and216 not currently used. If one day we need more bits, then we can use217 an invalid combination of hints to mean that another, larger field218 should be used in a different flag. */219 KMP_IDENT_ATOMIC_HINT_MASK = 0xFF0000,220 KMP_IDENT_ATOMIC_HINT_UNCONTENDED = 0x010000,221 KMP_IDENT_ATOMIC_HINT_CONTENDED = 0x020000,222 KMP_IDENT_ATOMIC_HINT_NONSPECULATIVE = 0x040000,223 KMP_IDENT_ATOMIC_HINT_SPECULATIVE = 0x080000,224 KMP_IDENT_OPENMP_SPEC_VERSION_MASK = 0xFF000000225};226 227/*!228 * The ident structure that describes a source location.229 */230typedef struct ident {231 kmp_int32 reserved_1; /**< might be used in Fortran; see above */232 kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags; KMP_IDENT_KMPC233 identifies this union member */234 kmp_int32 reserved_2; /**< not really used in Fortran any more; see above */235#if USE_ITT_BUILD236/* but currently used for storing region-specific ITT */237/* contextual information. */238#endif /* USE_ITT_BUILD */239 kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for C++ */240 char const *psource; /**< String describing the source location.241 The string is composed of semi-colon separated fields242 which describe the source file, the function and a pair243 of line numbers that delimit the construct. */244 // Returns the OpenMP version in form major*10+minor (e.g., 50 for 5.0)245 kmp_int32 get_openmp_version() {246 return (((flags & KMP_IDENT_OPENMP_SPEC_VERSION_MASK) >> 24) & 0xFF);247 }248} ident_t;249/*!250@}251*/252 253// Some forward declarations.254typedef union kmp_team kmp_team_t;255typedef struct kmp_taskdata kmp_taskdata_t;256typedef union kmp_task_team kmp_task_team_t;257typedef union kmp_team kmp_team_p;258typedef union kmp_info kmp_info_p;259typedef union kmp_root kmp_root_p;260 261template <bool C = false, bool S = true> class kmp_flag_32;262template <bool C = false, bool S = true> class kmp_flag_64;263template <bool C = false, bool S = true> class kmp_atomic_flag_64;264class kmp_flag_oncore;265 266#ifdef __cplusplus267extern "C" {268#endif269 270/* ------------------------------------------------------------------------ */271 272/* Pack two 32-bit signed integers into a 64-bit signed integer */273/* ToDo: Fix word ordering for big-endian machines. */274#define KMP_PACK_64(HIGH_32, LOW_32) \275 ((kmp_int64)((((kmp_uint64)(HIGH_32)) << 32) | (kmp_uint64)(LOW_32)))276 277// Generic string manipulation macros. Assume that _x is of type char *278#define SKIP_WS(_x) \279 { \280 while (*(_x) == ' ' || *(_x) == '\t') \281 (_x)++; \282 }283#define SKIP_DIGITS(_x) \284 { \285 while (*(_x) >= '0' && *(_x) <= '9') \286 (_x)++; \287 }288#define SKIP_TOKEN(_x) \289 { \290 while ((*(_x) >= '0' && *(_x) <= '9') || (*(_x) >= 'a' && *(_x) <= 'z') || \291 (*(_x) >= 'A' && *(_x) <= 'Z') || *(_x) == '_') \292 (_x)++; \293 }294#define SKIP_TO(_x, _c) \295 { \296 while (*(_x) != '\0' && *(_x) != (_c)) \297 (_x)++; \298 }299 300/* ------------------------------------------------------------------------ */301 302#define KMP_MAX(x, y) ((x) > (y) ? (x) : (y))303#define KMP_MIN(x, y) ((x) < (y) ? (x) : (y))304 305/* ------------------------------------------------------------------------ */306/* Enumeration types */307 308enum kmp_state_timer {309 ts_stop,310 ts_start,311 ts_pause,312 313 ts_last_state314};315 316enum dynamic_mode {317 dynamic_default,318#ifdef USE_LOAD_BALANCE319 dynamic_load_balance,320#endif /* USE_LOAD_BALANCE */321 dynamic_random,322 dynamic_thread_limit,323 dynamic_max324};325 326/* external schedule constants, duplicate enum omp_sched in omp.h in order to327 * not include it here */328#ifndef KMP_SCHED_TYPE_DEFINED329#define KMP_SCHED_TYPE_DEFINED330typedef enum kmp_sched {331 kmp_sched_lower = 0, // lower and upper bounds are for routine parameter check332 // Note: need to adjust __kmp_sch_map global array in case enum is changed333 kmp_sched_static = 1, // mapped to kmp_sch_static_chunked (33)334 kmp_sched_dynamic = 2, // mapped to kmp_sch_dynamic_chunked (35)335 kmp_sched_guided = 3, // mapped to kmp_sch_guided_chunked (36)336 kmp_sched_auto = 4, // mapped to kmp_sch_auto (38)337 kmp_sched_upper_std = 5, // upper bound for standard schedules338 kmp_sched_lower_ext = 100, // lower bound of Intel extension schedules339 kmp_sched_trapezoidal = 101, // mapped to kmp_sch_trapezoidal (39)340#if KMP_STATIC_STEAL_ENABLED341 kmp_sched_static_steal = 102, // mapped to kmp_sch_static_steal (44)342#endif343 kmp_sched_upper,344 kmp_sched_default = kmp_sched_static, // default scheduling345 kmp_sched_monotonic = 0x80000000346} kmp_sched_t;347#endif348 349/*!350 @ingroup WORK_SHARING351 * Describes the loop schedule to be used for a parallel for loop.352 */353enum sched_type : kmp_int32 {354 kmp_sch_lower = 32, /**< lower bound for unordered values */355 kmp_sch_static_chunked = 33,356 kmp_sch_static = 34, /**< static unspecialized */357 kmp_sch_dynamic_chunked = 35,358 kmp_sch_guided_chunked = 36, /**< guided unspecialized */359 kmp_sch_runtime = 37,360 kmp_sch_auto = 38, /**< auto */361 kmp_sch_trapezoidal = 39,362 363 /* accessible only through KMP_SCHEDULE environment variable */364 kmp_sch_static_greedy = 40,365 kmp_sch_static_balanced = 41,366 /* accessible only through KMP_SCHEDULE environment variable */367 kmp_sch_guided_iterative_chunked = 42,368 kmp_sch_guided_analytical_chunked = 43,369 /* accessible only through KMP_SCHEDULE environment variable */370 kmp_sch_static_steal = 44,371 372 /* static with chunk adjustment (e.g., simd) */373 kmp_sch_static_balanced_chunked = 45,374 kmp_sch_guided_simd = 46, /**< guided with chunk adjustment */375 kmp_sch_runtime_simd = 47, /**< runtime with chunk adjustment */376 377 /* accessible only through KMP_SCHEDULE environment variable */378 kmp_sch_upper, /**< upper bound for unordered values */379 380 kmp_ord_lower = 64, /**< lower bound for ordered values, must be power of 2 */381 kmp_ord_static_chunked = 65,382 kmp_ord_static = 66, /**< ordered static unspecialized */383 kmp_ord_dynamic_chunked = 67,384 kmp_ord_guided_chunked = 68,385 kmp_ord_runtime = 69,386 kmp_ord_auto = 70, /**< ordered auto */387 kmp_ord_trapezoidal = 71,388 kmp_ord_upper, /**< upper bound for ordered values */389 390 /* Schedules for Distribute construct */391 kmp_distribute_static_chunked = 91, /**< distribute static chunked */392 kmp_distribute_static = 92, /**< distribute static unspecialized */393 394 /* For the "nomerge" versions, kmp_dispatch_next*() will always return a395 single iteration/chunk, even if the loop is serialized. For the schedule396 types listed above, the entire iteration vector is returned if the loop is397 serialized. This doesn't work for gcc/gcomp sections. */398 kmp_nm_lower = 160, /**< lower bound for nomerge values */399 400 kmp_nm_static_chunked =401 (kmp_sch_static_chunked - kmp_sch_lower + kmp_nm_lower),402 kmp_nm_static = 162, /**< static unspecialized */403 kmp_nm_dynamic_chunked = 163,404 kmp_nm_guided_chunked = 164, /**< guided unspecialized */405 kmp_nm_runtime = 165,406 kmp_nm_auto = 166, /**< auto */407 kmp_nm_trapezoidal = 167,408 409 /* accessible only through KMP_SCHEDULE environment variable */410 kmp_nm_static_greedy = 168,411 kmp_nm_static_balanced = 169,412 /* accessible only through KMP_SCHEDULE environment variable */413 kmp_nm_guided_iterative_chunked = 170,414 kmp_nm_guided_analytical_chunked = 171,415 kmp_nm_static_steal =416 172, /* accessible only through OMP_SCHEDULE environment variable */417 418 kmp_nm_ord_static_chunked = 193,419 kmp_nm_ord_static = 194, /**< ordered static unspecialized */420 kmp_nm_ord_dynamic_chunked = 195,421 kmp_nm_ord_guided_chunked = 196,422 kmp_nm_ord_runtime = 197,423 kmp_nm_ord_auto = 198, /**< auto */424 kmp_nm_ord_trapezoidal = 199,425 kmp_nm_upper, /**< upper bound for nomerge values */426 427 /* Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers. Since428 we need to distinguish the three possible cases (no modifier, monotonic429 modifier, nonmonotonic modifier), we need separate bits for each modifier.430 The absence of monotonic does not imply nonmonotonic, especially since 4.5431 says that the behaviour of the "no modifier" case is implementation defined432 in 4.5, but will become "nonmonotonic" in 5.0.433 434 Since we're passing a full 32 bit value, we can use a couple of high bits435 for these flags; out of paranoia we avoid the sign bit.436 437 These modifiers can be or-ed into non-static schedules by the compiler to438 pass the additional information. They will be stripped early in the439 processing in __kmp_dispatch_init when setting up schedules, so most of the440 code won't ever see schedules with these bits set. */441 kmp_sch_modifier_monotonic =442 (1 << 29), /**< Set if the monotonic schedule modifier was present */443 kmp_sch_modifier_nonmonotonic =444 (1 << 30), /**< Set if the nonmonotonic schedule modifier was present */445 446#define SCHEDULE_WITHOUT_MODIFIERS(s) \447 (enum sched_type)( \448 (s) & ~(kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic))449#define SCHEDULE_HAS_MONOTONIC(s) (((s)&kmp_sch_modifier_monotonic) != 0)450#define SCHEDULE_HAS_NONMONOTONIC(s) (((s)&kmp_sch_modifier_nonmonotonic) != 0)451#define SCHEDULE_HAS_NO_MODIFIERS(s) \452 (((s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)) == 0)453#define SCHEDULE_GET_MODIFIERS(s) \454 ((enum sched_type)( \455 (s) & (kmp_sch_modifier_nonmonotonic | kmp_sch_modifier_monotonic)))456#define SCHEDULE_SET_MODIFIERS(s, m) \457 (s = (enum sched_type)((kmp_int32)s | (kmp_int32)m))458#define SCHEDULE_NONMONOTONIC 0459#define SCHEDULE_MONOTONIC 1460 461 kmp_sch_default = kmp_sch_static /**< default scheduling algorithm */462};463 464// Apply modifiers on internal kind to standard kind465static inline void466__kmp_sched_apply_mods_stdkind(kmp_sched_t *kind,467 enum sched_type internal_kind) {468 if (SCHEDULE_HAS_MONOTONIC(internal_kind)) {469 *kind = (kmp_sched_t)((int)*kind | (int)kmp_sched_monotonic);470 }471}472 473// Apply modifiers on standard kind to internal kind474static inline void475__kmp_sched_apply_mods_intkind(kmp_sched_t kind,476 enum sched_type *internal_kind) {477 if ((int)kind & (int)kmp_sched_monotonic) {478 *internal_kind = (enum sched_type)((int)*internal_kind |479 (int)kmp_sch_modifier_monotonic);480 }481}482 483// Get standard schedule without modifiers484static inline kmp_sched_t __kmp_sched_without_mods(kmp_sched_t kind) {485 return (kmp_sched_t)((int)kind & ~((int)kmp_sched_monotonic));486}487 488/* Type to keep runtime schedule set via OMP_SCHEDULE or omp_set_schedule() */489typedef union kmp_r_sched {490 struct {491 enum sched_type r_sched_type;492 int chunk;493 };494 kmp_int64 sched;495} kmp_r_sched_t;496 497extern enum sched_type __kmp_sch_map[]; // map OMP 3.0 schedule types with our498// internal schedule types499 500enum library_type {501 library_none,502 library_serial,503 library_turnaround,504 library_throughput505};506 507#if KMP_MIC_SUPPORTED508enum mic_type { non_mic, mic1, mic2, mic3, dummy };509#endif510 511// OpenMP 3.1 - Nested num threads array512typedef struct kmp_nested_nthreads_t {513 int *nth;514 int size;515 int used;516} kmp_nested_nthreads_t;517 518extern kmp_nested_nthreads_t __kmp_nested_nth;519 520/* -- fast reduction stuff ------------------------------------------------ */521 522#undef KMP_FAST_REDUCTION_BARRIER523#define KMP_FAST_REDUCTION_BARRIER 1524 525#undef KMP_FAST_REDUCTION_CORE_DUO526#if KMP_ARCH_X86 || KMP_ARCH_X86_64527#define KMP_FAST_REDUCTION_CORE_DUO 1528#endif529 530enum _reduction_method {531 reduction_method_not_defined = 0,532 critical_reduce_block = (1 << 8),533 atomic_reduce_block = (2 << 8),534 tree_reduce_block = (3 << 8),535 empty_reduce_block = (4 << 8)536};537 538// Description of the packed_reduction_method variable:539// The packed_reduction_method variable consists of two enum types variables540// that are packed together into 0-th byte and 1-st byte:541// 0: (packed_reduction_method & 0x000000FF) is a 'enum barrier_type' value of542// barrier that will be used in fast reduction: bs_plain_barrier or543// bs_reduction_barrier544// 1: (packed_reduction_method & 0x0000FF00) is a reduction method that will545// be used in fast reduction;546// Reduction method is of 'enum _reduction_method' type and it's defined the way547// so that the bits of 0-th byte are empty, so no need to execute a shift548// instruction while packing/unpacking549 550#if KMP_FAST_REDUCTION_BARRIER551#define PACK_REDUCTION_METHOD_AND_BARRIER(reduction_method, barrier_type) \552 ((reduction_method) | (barrier_type))553 554#define UNPACK_REDUCTION_METHOD(packed_reduction_method) \555 ((enum _reduction_method)((packed_reduction_method) & (0x0000FF00)))556 557#define UNPACK_REDUCTION_BARRIER(packed_reduction_method) \558 ((enum barrier_type)((packed_reduction_method) & (0x000000FF)))559#else560#define PACK_REDUCTION_METHOD_AND_BARRIER(reduction_method, barrier_type) \561 (reduction_method)562 563#define UNPACK_REDUCTION_METHOD(packed_reduction_method) \564 (packed_reduction_method)565 566#define UNPACK_REDUCTION_BARRIER(packed_reduction_method) (bs_plain_barrier)567#endif568 569#define TEST_REDUCTION_METHOD(packed_reduction_method, which_reduction_block) \570 ((UNPACK_REDUCTION_METHOD(packed_reduction_method)) == \571 (which_reduction_block))572 573#if KMP_FAST_REDUCTION_BARRIER574#define TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER \575 (PACK_REDUCTION_METHOD_AND_BARRIER(tree_reduce_block, bs_reduction_barrier))576 577#define TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER \578 (PACK_REDUCTION_METHOD_AND_BARRIER(tree_reduce_block, bs_plain_barrier))579#endif580 581typedef int PACKED_REDUCTION_METHOD_T;582 583/* -- end of fast reduction stuff ----------------------------------------- */584 585#if KMP_OS_WINDOWS586#define USE_CBLKDATA587#if KMP_MSVC_COMPAT588#pragma warning(push)589#pragma warning(disable : 271 310)590#endif591#include <windows.h>592#if KMP_MSVC_COMPAT593#pragma warning(pop)594#endif595#endif596 597#if KMP_OS_UNIX598#if !KMP_OS_WASI599#include <dlfcn.h>600#endif601#include <pthread.h>602#endif603 604enum kmp_hw_t : int {605 KMP_HW_UNKNOWN = -1,606 KMP_HW_SOCKET = 0,607 KMP_HW_PROC_GROUP,608 KMP_HW_NUMA,609 KMP_HW_DIE,610 KMP_HW_LLC,611 KMP_HW_L3,612 KMP_HW_TILE,613 KMP_HW_MODULE,614 KMP_HW_L2,615 KMP_HW_L1,616 KMP_HW_CORE,617 KMP_HW_THREAD,618 KMP_HW_LAST619};620 621typedef enum kmp_hw_core_type_t {622 KMP_HW_CORE_TYPE_UNKNOWN = 0x0,623#if KMP_ARCH_X86 || KMP_ARCH_X86_64624 KMP_HW_CORE_TYPE_ATOM = 0x20,625 KMP_HW_CORE_TYPE_CORE = 0x40,626 KMP_HW_MAX_NUM_CORE_TYPES = 3,627#else628 KMP_HW_MAX_NUM_CORE_TYPES = 1,629#endif630} kmp_hw_core_type_t;631 632#define KMP_HW_MAX_NUM_CORE_EFFS 8633 634#define KMP_DEBUG_ASSERT_VALID_HW_TYPE(type) \635 KMP_DEBUG_ASSERT(type >= (kmp_hw_t)0 && type < KMP_HW_LAST)636#define KMP_ASSERT_VALID_HW_TYPE(type) \637 KMP_ASSERT(type >= (kmp_hw_t)0 && type < KMP_HW_LAST)638 639#define KMP_FOREACH_HW_TYPE(type) \640 for (kmp_hw_t type = (kmp_hw_t)0; type < KMP_HW_LAST; \641 type = (kmp_hw_t)((int)type + 1))642 643const char *__kmp_hw_get_keyword(kmp_hw_t type, bool plural = false);644const char *__kmp_hw_get_catalog_string(kmp_hw_t type, bool plural = false);645const char *__kmp_hw_get_core_type_string(kmp_hw_core_type_t type);646 647/* Only Linux* OS and Windows* OS support thread affinity. */648#if KMP_AFFINITY_SUPPORTED649 650// GROUP_AFFINITY is already defined for _MSC_VER>=1600 (VS2010 and later).651#if KMP_OS_WINDOWS652#if _MSC_VER < 1600 && KMP_MSVC_COMPAT653typedef struct GROUP_AFFINITY {654 KAFFINITY Mask;655 WORD Group;656 WORD Reserved[3];657} GROUP_AFFINITY;658#endif /* _MSC_VER < 1600 */659#if KMP_GROUP_AFFINITY660extern int __kmp_num_proc_groups;661#else662static const int __kmp_num_proc_groups = 1;663#endif /* KMP_GROUP_AFFINITY */664typedef DWORD (*kmp_GetActiveProcessorCount_t)(WORD);665extern kmp_GetActiveProcessorCount_t __kmp_GetActiveProcessorCount;666 667typedef WORD (*kmp_GetActiveProcessorGroupCount_t)(void);668extern kmp_GetActiveProcessorGroupCount_t __kmp_GetActiveProcessorGroupCount;669 670typedef BOOL (*kmp_GetThreadGroupAffinity_t)(HANDLE, GROUP_AFFINITY *);671extern kmp_GetThreadGroupAffinity_t __kmp_GetThreadGroupAffinity;672 673typedef BOOL (*kmp_SetThreadGroupAffinity_t)(HANDLE, const GROUP_AFFINITY *,674 GROUP_AFFINITY *);675extern kmp_SetThreadGroupAffinity_t __kmp_SetThreadGroupAffinity;676#endif /* KMP_OS_WINDOWS */677 678#if KMP_HWLOC_ENABLED679extern hwloc_topology_t __kmp_hwloc_topology;680extern int __kmp_hwloc_error;681#endif // KMP_HWLOC_ENABLED682 683extern size_t __kmp_affin_mask_size;684#define KMP_AFFINITY_CAPABLE() (__kmp_affin_mask_size > 0)685#define KMP_AFFINITY_DISABLE() (__kmp_affin_mask_size = 0)686#define KMP_AFFINITY_ENABLE(mask_size) (__kmp_affin_mask_size = mask_size)687#define KMP_CPU_SET_ITERATE(i, mask) \688 for (i = (mask)->begin(); (int)i != (mask)->end(); i = (mask)->next(i))689#define KMP_CPU_SET(i, mask) (mask)->set(i)690#define KMP_CPU_ISSET(i, mask) (mask)->is_set(i)691#define KMP_CPU_CLR(i, mask) (mask)->clear(i)692#define KMP_CPU_ZERO(mask) (mask)->zero()693#define KMP_CPU_ISEMPTY(mask) (mask)->empty()694#define KMP_CPU_COPY(dest, src) (dest)->copy(src)695#define KMP_CPU_AND(dest, src) (dest)->bitwise_and(src)696#define KMP_CPU_COMPLEMENT(max_bit_number, mask) (mask)->bitwise_not()697#define KMP_CPU_UNION(dest, src) (dest)->bitwise_or(src)698#define KMP_CPU_EQUAL(dest, src) (dest)->is_equal(src)699#define KMP_CPU_ALLOC(ptr) (ptr = __kmp_affinity_dispatch->allocate_mask())700#define KMP_CPU_FREE(ptr) __kmp_affinity_dispatch->deallocate_mask(ptr)701#define KMP_CPU_ALLOC_ON_STACK(ptr) KMP_CPU_ALLOC(ptr)702#define KMP_CPU_FREE_FROM_STACK(ptr) KMP_CPU_FREE(ptr)703#define KMP_CPU_INTERNAL_ALLOC(ptr) KMP_CPU_ALLOC(ptr)704#define KMP_CPU_INTERNAL_FREE(ptr) KMP_CPU_FREE(ptr)705#define KMP_CPU_INDEX(arr, i) __kmp_affinity_dispatch->index_mask_array(arr, i)706#define KMP_CPU_ALLOC_ARRAY(arr, n) \707 (arr = __kmp_affinity_dispatch->allocate_mask_array(n))708#define KMP_CPU_FREE_ARRAY(arr, n) \709 __kmp_affinity_dispatch->deallocate_mask_array(arr)710#define KMP_CPU_INTERNAL_ALLOC_ARRAY(arr, n) KMP_CPU_ALLOC_ARRAY(arr, n)711#define KMP_CPU_INTERNAL_FREE_ARRAY(arr, n) KMP_CPU_FREE_ARRAY(arr, n)712#define __kmp_get_system_affinity(mask, abort_bool) \713 (mask)->get_system_affinity(abort_bool)714#define __kmp_set_system_affinity(mask, abort_bool) \715 (mask)->set_system_affinity(abort_bool)716#define __kmp_get_proc_group(mask) (mask)->get_proc_group()717 718class KMPAffinity {719public:720 class Mask {721 public:722 void *operator new(size_t n);723 void operator delete(void *p);724 void *operator new[](size_t n);725 void operator delete[](void *p);726 virtual ~Mask() {}727 // Set bit i to 1728 virtual void set(int i) {}729 // Return bit i730 virtual bool is_set(int i) const { return false; }731 // Set bit i to 0732 virtual void clear(int i) {}733 // Zero out entire mask734 virtual void zero() {}735 // Check whether mask is empty736 virtual bool empty() const { return true; }737 // Copy src into this mask738 virtual void copy(const Mask *src) {}739 // this &= rhs740 virtual void bitwise_and(const Mask *rhs) {}741 // this |= rhs742 virtual void bitwise_or(const Mask *rhs) {}743 // this = ~this744 virtual void bitwise_not() {}745 // this == rhs746 virtual bool is_equal(const Mask *rhs) const { return false; }747 // API for iterating over an affinity mask748 // for (int i = mask->begin(); i != mask->end(); i = mask->next(i))749 virtual int begin() const { return 0; }750 virtual int end() const { return 0; }751 virtual int next(int previous) const { return 0; }752#if KMP_OS_WINDOWS753 virtual int set_process_affinity(bool abort_on_error) const { return -1; }754#endif755 // Set the system's affinity to this affinity mask's value756 virtual int set_system_affinity(bool abort_on_error) const { return -1; }757 // Set this affinity mask to the current system affinity758 virtual int get_system_affinity(bool abort_on_error) { return -1; }759 // Only 1 DWORD in the mask should have any procs set.760 // Return the appropriate index, or -1 for an invalid mask.761 virtual int get_proc_group() const { return -1; }762 int get_max_cpu() const {763 int cpu;764 int max_cpu = -1;765 KMP_CPU_SET_ITERATE(cpu, this) {766 if (cpu > max_cpu)767 max_cpu = cpu;768 }769 return max_cpu;770 }771 };772 void *operator new(size_t n);773 void operator delete(void *p);774 // Need virtual destructor775 virtual ~KMPAffinity() = default;776 // Determine if affinity is capable777 virtual void determine_capable(const char *env_var) {}778 // Bind the current thread to os proc779 virtual void bind_thread(int proc) {}780 // Factory functions to allocate/deallocate a mask781 virtual Mask *allocate_mask() { return nullptr; }782 virtual void deallocate_mask(Mask *m) {}783 virtual Mask *allocate_mask_array(int num) { return nullptr; }784 virtual void deallocate_mask_array(Mask *m) {}785 virtual Mask *index_mask_array(Mask *m, int index) { return nullptr; }786 static void pick_api();787 static void destroy_api();788 enum api_type {789 NATIVE_OS790#if KMP_HWLOC_ENABLED791 ,792 HWLOC793#endif // KMP_HWLOC_ENABLED794 };795 virtual api_type get_api_type() const {796 KMP_ASSERT(0);797 return NATIVE_OS;798 }799 800private:801 static bool picked_api;802};803 804typedef KMPAffinity::Mask kmp_affin_mask_t;805extern KMPAffinity *__kmp_affinity_dispatch;806 807#if !KMP_OS_AIX808class kmp_affinity_raii_t {809 kmp_affin_mask_t *mask;810 bool restored;811 812public:813 kmp_affinity_raii_t(const kmp_affin_mask_t *new_mask = nullptr)814 : mask(nullptr), restored(false) {815 if (KMP_AFFINITY_CAPABLE()) {816 KMP_CPU_ALLOC(mask);817 KMP_ASSERT(mask != NULL);818 __kmp_get_system_affinity(mask, /*abort_on_error=*/true);819 if (new_mask)820 __kmp_set_system_affinity(new_mask, /*abort_on_error=*/true);821 }822 }823 void restore() {824 if (mask && KMP_AFFINITY_CAPABLE() && !restored) {825 __kmp_set_system_affinity(mask, /*abort_on_error=*/true);826 KMP_CPU_FREE(mask);827 }828 restored = true;829 }830 ~kmp_affinity_raii_t() { restore(); }831};832#endif // !KMP_OS_AIX833 834// Declare local char buffers with this size for printing debug and info835// messages, using __kmp_affinity_print_mask().836#define KMP_AFFIN_MASK_PRINT_LEN 1024837 838enum affinity_type {839 affinity_none = 0,840 affinity_physical,841 affinity_logical,842 affinity_compact,843 affinity_scatter,844 affinity_explicit,845 affinity_balanced,846 affinity_disabled, // not used outsize the env var parser847 affinity_default848};849 850enum affinity_top_method {851 affinity_top_method_all = 0, // try all (supported) methods, in order852#if KMP_ARCH_X86 || KMP_ARCH_X86_64853 affinity_top_method_apicid,854 affinity_top_method_x2apicid,855 affinity_top_method_x2apicid_1f,856#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */857 affinity_top_method_cpuinfo, // KMP_CPUINFO_FILE is usable on Windows* OS, too858#if KMP_GROUP_AFFINITY859 affinity_top_method_group,860#endif /* KMP_GROUP_AFFINITY */861 affinity_top_method_flat,862#if KMP_HWLOC_ENABLED863 affinity_top_method_hwloc,864#endif // KMP_HWLOC_ENABLED865 affinity_top_method_default866};867 868#define affinity_respect_mask_default (2)869 870typedef struct kmp_affinity_flags_t {871 unsigned dups : 1;872 unsigned verbose : 1;873 unsigned warnings : 1;874 unsigned respect : 2;875 unsigned reset : 1;876 unsigned initialized : 1;877 unsigned core_types_gran : 1;878 unsigned core_effs_gran : 1;879 unsigned omp_places : 1;880 unsigned reserved : 22;881} kmp_affinity_flags_t;882KMP_BUILD_ASSERT(sizeof(kmp_affinity_flags_t) == 4);883 884typedef struct kmp_affinity_ids_t {885 int os_id;886 int ids[KMP_HW_LAST];887} kmp_affinity_ids_t;888 889typedef struct kmp_affinity_attrs_t {890 int core_type : 8;891 int core_eff : 8;892 unsigned valid : 1;893 unsigned reserved : 15;894} kmp_affinity_attrs_t;895#define KMP_AFFINITY_ATTRS_UNKNOWN \896 { KMP_HW_CORE_TYPE_UNKNOWN, kmp_hw_attr_t::UNKNOWN_CORE_EFF, 0, 0 }897 898typedef struct kmp_affinity_t {899 char *proclist;900 enum affinity_type type;901 kmp_hw_t gran;902 int gran_levels;903 kmp_affinity_attrs_t core_attr_gran;904 int compact;905 int offset;906 kmp_affinity_flags_t flags;907 unsigned num_masks;908 kmp_affin_mask_t *masks;909 kmp_affinity_ids_t *ids;910 kmp_affinity_attrs_t *attrs;911 unsigned num_os_id_masks;912 kmp_affin_mask_t *os_id_masks;913 const char *env_var;914} kmp_affinity_t;915 916#define KMP_AFFINITY_INIT(env) \917 { \918 nullptr, affinity_default, KMP_HW_UNKNOWN, -1, KMP_AFFINITY_ATTRS_UNKNOWN, \919 0, 0, \920 {TRUE, FALSE, TRUE, affinity_respect_mask_default, FALSE, FALSE, \921 FALSE, FALSE, FALSE}, \922 0, nullptr, nullptr, nullptr, 0, nullptr, env \923 }924 925extern enum affinity_top_method __kmp_affinity_top_method;926extern kmp_affinity_t __kmp_affinity;927extern kmp_affinity_t __kmp_hh_affinity;928extern kmp_affinity_t *__kmp_affinities[2];929 930extern void __kmp_affinity_bind_thread(int which);931 932extern kmp_affin_mask_t *__kmp_affin_fullMask;933extern kmp_affin_mask_t *__kmp_affin_origMask;934extern char *__kmp_cpuinfo_file;935 936#if KMP_WEIGHTED_ITERATIONS_SUPPORTED937extern int __kmp_first_osid_with_ecore;938#endif939 940#endif /* KMP_AFFINITY_SUPPORTED */941 942// This needs to be kept in sync with the values in omp.h !!!943typedef enum kmp_proc_bind_t {944 proc_bind_false = 0,945 proc_bind_true,946 proc_bind_primary,947 proc_bind_close,948 proc_bind_spread,949 proc_bind_intel, // use KMP_AFFINITY interface950 proc_bind_default951} kmp_proc_bind_t;952 953typedef struct kmp_nested_proc_bind_t {954 kmp_proc_bind_t *bind_types;955 int size;956 int used;957} kmp_nested_proc_bind_t;958 959extern kmp_nested_proc_bind_t __kmp_nested_proc_bind;960extern kmp_proc_bind_t __kmp_teams_proc_bind;961 962extern int __kmp_display_affinity;963extern char *__kmp_affinity_format;964static const size_t KMP_AFFINITY_FORMAT_SIZE = 512;965#if OMPT_SUPPORT966extern int __kmp_tool;967extern char *__kmp_tool_libraries;968#endif // OMPT_SUPPORT969 970#if KMP_AFFINITY_SUPPORTED971#define KMP_PLACE_ALL (-1)972#define KMP_PLACE_UNDEFINED (-2)973// Is KMP_AFFINITY is being used instead of OMP_PROC_BIND/OMP_PLACES?974#define KMP_AFFINITY_NON_PROC_BIND \975 ((__kmp_nested_proc_bind.bind_types[0] == proc_bind_false || \976 __kmp_nested_proc_bind.bind_types[0] == proc_bind_intel) && \977 (__kmp_affinity.num_masks > 0 || __kmp_affinity.type == affinity_balanced))978#endif /* KMP_AFFINITY_SUPPORTED */979 980extern int __kmp_affinity_num_places;981 982typedef enum kmp_cancel_kind_t {983 cancel_noreq = 0,984 cancel_parallel = 1,985 cancel_loop = 2,986 cancel_sections = 3,987 cancel_taskgroup = 4988} kmp_cancel_kind_t;989 990// KMP_HW_SUBSET support:991typedef struct kmp_hws_item {992 int num;993 int offset;994} kmp_hws_item_t;995 996extern kmp_hws_item_t __kmp_hws_socket;997extern kmp_hws_item_t __kmp_hws_die;998extern kmp_hws_item_t __kmp_hws_node;999extern kmp_hws_item_t __kmp_hws_tile;1000extern kmp_hws_item_t __kmp_hws_core;1001extern kmp_hws_item_t __kmp_hws_proc;1002extern int __kmp_hws_requested;1003extern int __kmp_hws_abs_flag; // absolute or per-item number requested1004 1005/* ------------------------------------------------------------------------ */1006 1007#define KMP_PAD(type, sz) \1008 (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1))1009 1010// We need to avoid using -1 as a GTID as +1 is added to the gtid1011// when storing it in a lock, and the value 0 is reserved.1012#define KMP_GTID_DNE (-2) /* Does not exist */1013#define KMP_GTID_SHUTDOWN (-3) /* Library is shutting down */1014#define KMP_GTID_MONITOR (-4) /* Monitor thread ID */1015#define KMP_GTID_UNKNOWN (-5) /* Is not known */1016#define KMP_GTID_MIN (-6) /* Minimal gtid for low bound check in DEBUG */1017 1018/* OpenMP 5.0 Memory Management support */1019 1020#ifndef __OMP_H1021// Duplicate type definitions from omp.h1022typedef uintptr_t omp_uintptr_t;1023 1024typedef enum {1025 omp_atk_sync_hint = 1,1026 omp_atk_alignment = 2,1027 omp_atk_access = 3,1028 omp_atk_pool_size = 4,1029 omp_atk_fallback = 5,1030 omp_atk_fb_data = 6,1031 omp_atk_pinned = 7,1032 omp_atk_partition = 8,1033 omp_atk_pin_device = 9,1034 omp_atk_preferred_device = 10,1035 omp_atk_device_access = 11,1036 omp_atk_target_access = 12,1037 omp_atk_atomic_scope = 13,1038 omp_atk_part_size = 141039} omp_alloctrait_key_t;1040 1041typedef enum {1042 omp_atv_false = 0,1043 omp_atv_true = 1,1044 omp_atv_contended = 3,1045 omp_atv_uncontended = 4,1046 omp_atv_serialized = 5,1047 omp_atv_sequential = omp_atv_serialized, // (deprecated)1048 omp_atv_private = 6,1049 omp_atv_device = 7,1050 omp_atv_thread = 8,1051 omp_atv_pteam = 9,1052 omp_atv_cgroup = 10,1053 omp_atv_default_mem_fb = 11,1054 omp_atv_null_fb = 12,1055 omp_atv_abort_fb = 13,1056 omp_atv_allocator_fb = 14,1057 omp_atv_environment = 15,1058 omp_atv_nearest = 16,1059 omp_atv_blocked = 17,1060 omp_atv_interleaved = 18,1061 omp_atv_all = 19,1062 omp_atv_single = 20,1063 omp_atv_multiple = 21,1064 omp_atv_memspace = 221065} omp_alloctrait_value_t;1066#define omp_atv_default ((omp_uintptr_t)-1)1067 1068typedef void *omp_memspace_handle_t;1069extern omp_memspace_handle_t const omp_null_mem_space;1070extern omp_memspace_handle_t const omp_default_mem_space;1071extern omp_memspace_handle_t const omp_large_cap_mem_space;1072extern omp_memspace_handle_t const omp_const_mem_space;1073extern omp_memspace_handle_t const omp_high_bw_mem_space;1074extern omp_memspace_handle_t const omp_low_lat_mem_space;1075extern omp_memspace_handle_t const llvm_omp_target_host_mem_space;1076extern omp_memspace_handle_t const llvm_omp_target_shared_mem_space;1077extern omp_memspace_handle_t const llvm_omp_target_device_mem_space;1078extern omp_memspace_handle_t const kmp_max_mem_space;1079 1080typedef struct {1081 omp_alloctrait_key_t key;1082 omp_uintptr_t value;1083} omp_alloctrait_t;1084 1085typedef void *omp_allocator_handle_t;1086extern omp_allocator_handle_t const omp_null_allocator;1087extern omp_allocator_handle_t const omp_default_mem_alloc;1088extern omp_allocator_handle_t const omp_large_cap_mem_alloc;1089extern omp_allocator_handle_t const omp_const_mem_alloc;1090extern omp_allocator_handle_t const omp_high_bw_mem_alloc;1091extern omp_allocator_handle_t const omp_low_lat_mem_alloc;1092extern omp_allocator_handle_t const omp_cgroup_mem_alloc;1093extern omp_allocator_handle_t const omp_pteam_mem_alloc;1094extern omp_allocator_handle_t const omp_thread_mem_alloc;1095extern omp_allocator_handle_t const llvm_omp_target_host_mem_alloc;1096extern omp_allocator_handle_t const llvm_omp_target_shared_mem_alloc;1097extern omp_allocator_handle_t const llvm_omp_target_device_mem_alloc;1098extern omp_allocator_handle_t const kmp_max_mem_alloc;1099extern omp_allocator_handle_t __kmp_def_allocator;1100 1101// end of duplicate type definitions from omp.h1102#endif1103 1104extern int __kmp_memkind_available;1105extern bool __kmp_hwloc_available;1106 1107/// Memory space informaition is shared with offload runtime.1108typedef struct kmp_memspace_t {1109 omp_memspace_handle_t memspace; // predefined input memory space1110 int num_resources = 0; // number of available resources1111 int *resources = nullptr; // available resources1112 kmp_memspace_t *next = nullptr; // next memory space handle1113} kmp_memspace_t;1114 1115/// Memory allocator information is shared with offload runtime.1116typedef struct kmp_allocator_t {1117 omp_memspace_handle_t memspace;1118 void **memkind; // pointer to memkind1119 size_t alignment;1120 omp_alloctrait_value_t fb;1121 kmp_allocator_t *fb_data;1122 kmp_uint64 pool_size;1123 kmp_uint64 pool_used;1124 bool pinned;1125 omp_alloctrait_value_t partition;1126 int pin_device;1127 int preferred_device;1128 omp_alloctrait_value_t target_access;1129 omp_alloctrait_value_t atomic_scope;1130 size_t part_size;1131#if KMP_HWLOC_ENABLED1132 omp_alloctrait_value_t membind;1133#endif // KMP_HWLOC_ENABLED1134} kmp_allocator_t;1135 1136extern omp_allocator_handle_t __kmpc_init_allocator(int gtid,1137 omp_memspace_handle_t,1138 int ntraits,1139 omp_alloctrait_t traits[]);1140extern void __kmpc_destroy_allocator(int gtid, omp_allocator_handle_t al);1141extern void __kmpc_set_default_allocator(int gtid, omp_allocator_handle_t al);1142extern omp_allocator_handle_t __kmpc_get_default_allocator(int gtid);1143// external interfaces, may be used by compiler1144extern void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al);1145extern void *__kmpc_aligned_alloc(int gtid, size_t align, size_t sz,1146 omp_allocator_handle_t al);1147extern void *__kmpc_calloc(int gtid, size_t nmemb, size_t sz,1148 omp_allocator_handle_t al);1149extern void *__kmpc_realloc(int gtid, void *ptr, size_t sz,1150 omp_allocator_handle_t al,1151 omp_allocator_handle_t free_al);1152extern void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);1153// internal interfaces, contain real implementation1154extern void *__kmp_alloc(int gtid, size_t align, size_t sz,1155 omp_allocator_handle_t al);1156extern void *__kmp_calloc(int gtid, size_t align, size_t nmemb, size_t sz,1157 omp_allocator_handle_t al);1158extern void *__kmp_realloc(int gtid, void *ptr, size_t sz,1159 omp_allocator_handle_t al,1160 omp_allocator_handle_t free_al);1161extern void ___kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al);1162 1163extern void __kmp_init_memkind();1164extern void __kmp_fini_memkind();1165extern void __kmp_init_target_mem();1166extern void __kmp_fini_target_mem();1167 1168// OpenMP 6.0 (TR11) Memory Management support1169extern omp_memspace_handle_t __kmp_get_devices_memspace(int ndevs,1170 const int *devs,1171 omp_memspace_handle_t,1172 int host);1173extern omp_allocator_handle_t __kmp_get_devices_allocator(int ndevs,1174 const int *devs,1175 omp_memspace_handle_t,1176 int host);1177extern int __kmp_get_memspace_num_resources(omp_memspace_handle_t memspace);1178extern omp_memspace_handle_t1179__kmp_get_submemspace(omp_memspace_handle_t memspace, int num_resources,1180 int *resources);1181 1182/* ------------------------------------------------------------------------ */1183 1184#if ENABLE_LIBOMPTARGET1185extern void __kmp_init_target_task();1186#endif1187 1188/* ------------------------------------------------------------------------ */1189 1190#define KMP_UINT64_MAX \1191 (~((kmp_uint64)1 << ((sizeof(kmp_uint64) * (1 << 3)) - 1)))1192 1193#define KMP_MIN_NTH 11194 1195#ifndef KMP_MAX_NTH1196#if defined(PTHREAD_THREADS_MAX) && PTHREAD_THREADS_MAX < INT_MAX1197#define KMP_MAX_NTH PTHREAD_THREADS_MAX1198#else1199#ifdef __ve__1200// VE's pthread supports only up to 64 threads per a VE process.1201// Please check p. 14 of following documentation for more details.1202// https://sxauroratsubasa.sakura.ne.jp/documents/veos/en/VEOS_high_level_design.pdf1203#define KMP_MAX_NTH 641204#else1205#define KMP_MAX_NTH INT_MAX1206#endif1207#endif1208#endif /* KMP_MAX_NTH */1209 1210#ifdef PTHREAD_STACK_MIN1211#define KMP_MIN_STKSIZE ((size_t)PTHREAD_STACK_MIN)1212#else1213#define KMP_MIN_STKSIZE ((size_t)(32 * 1024))1214#endif1215 1216#if KMP_OS_AIX && KMP_ARCH_PPC1217#define KMP_MAX_STKSIZE 0x10000000 /* 256Mb max size on 32-bit AIX */1218#else1219#define KMP_MAX_STKSIZE (~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)))1220#endif1221 1222#if KMP_ARCH_X861223#define KMP_DEFAULT_STKSIZE ((size_t)(2 * 1024 * 1024))1224#elif KMP_ARCH_X86_641225#define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024))1226#define KMP_BACKUP_STKSIZE ((size_t)(2 * 1024 * 1024))1227#elif KMP_ARCH_VE1228// Minimum stack size for pthread for VE is 4MB.1229// https://www.hpc.nec/documents/veos/en/glibc/Difference_Points_glibc.htm1230#define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024))1231#elif KMP_OS_AIX1232// The default stack size for worker threads on AIX is 4MB.1233#define KMP_DEFAULT_STKSIZE ((size_t)(4 * 1024 * 1024))1234#else1235#define KMP_DEFAULT_STKSIZE ((size_t)(1024 * 1024))1236#endif1237 1238#define KMP_DEFAULT_MALLOC_POOL_INCR ((size_t)(1024 * 1024))1239#define KMP_MIN_MALLOC_POOL_INCR ((size_t)(4 * 1024))1240#define KMP_MAX_MALLOC_POOL_INCR \1241 (~((size_t)1 << ((sizeof(size_t) * (1 << 3)) - 1)))1242 1243#define KMP_MIN_STKOFFSET (0)1244#define KMP_MAX_STKOFFSET KMP_MAX_STKSIZE1245#if KMP_OS_DARWIN1246#define KMP_DEFAULT_STKOFFSET KMP_MIN_STKOFFSET1247#else1248#define KMP_DEFAULT_STKOFFSET CACHE_LINE1249#endif1250 1251#define KMP_MIN_STKPADDING (0)1252#define KMP_MAX_STKPADDING (2 * 1024 * 1024)1253 1254#define KMP_BLOCKTIME_MULTIPLIER \1255 (1000000) /* number of blocktime units per second */1256#define KMP_MIN_BLOCKTIME (0)1257#define KMP_MAX_BLOCKTIME \1258 (INT_MAX) /* Must be this for "infinite" setting the work */1259 1260/* __kmp_blocktime is in microseconds */1261#define KMP_DEFAULT_BLOCKTIME (__kmp_is_hybrid_cpu() ? (0) : (200000))1262 1263#if KMP_USE_MONITOR1264#define KMP_DEFAULT_MONITOR_STKSIZE ((size_t)(64 * 1024))1265#define KMP_MIN_MONITOR_WAKEUPS (1) // min times monitor wakes up per second1266#define KMP_MAX_MONITOR_WAKEUPS (1000) // max times monitor can wake up per sec1267 1268/* Calculate new number of monitor wakeups for a specific block time based on1269 previous monitor_wakeups. Only allow increasing number of wakeups */1270#define KMP_WAKEUPS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \1271 (((blocktime) == KMP_MAX_BLOCKTIME) ? (monitor_wakeups) \1272 : ((blocktime) == KMP_MIN_BLOCKTIME) ? KMP_MAX_MONITOR_WAKEUPS \1273 : ((monitor_wakeups) > (KMP_BLOCKTIME_MULTIPLIER / (blocktime))) \1274 ? (monitor_wakeups) \1275 : (KMP_BLOCKTIME_MULTIPLIER) / (blocktime))1276 1277/* Calculate number of intervals for a specific block time based on1278 monitor_wakeups */1279#define KMP_INTERVALS_FROM_BLOCKTIME(blocktime, monitor_wakeups) \1280 (((blocktime) + (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)) - 1) / \1281 (KMP_BLOCKTIME_MULTIPLIER / (monitor_wakeups)))1282#else1283#define KMP_BLOCKTIME(team, tid) \1284 (get__bt_set(team, tid) ? get__blocktime(team, tid) : __kmp_dflt_blocktime)1285#if KMP_OS_UNIX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)1286// HW TSC is used to reduce overhead (clock tick instead of nanosecond).1287extern kmp_uint64 __kmp_ticks_per_msec;1288extern kmp_uint64 __kmp_ticks_per_usec;1289#if KMP_COMPILER_ICC || KMP_COMPILER_ICX1290#define KMP_NOW() ((kmp_uint64)_rdtsc())1291#else1292#define KMP_NOW() __kmp_hardware_timestamp()1293#endif1294#define KMP_BLOCKTIME_INTERVAL(team, tid) \1295 ((kmp_uint64)KMP_BLOCKTIME(team, tid) * __kmp_ticks_per_usec)1296#define KMP_BLOCKING(goal, count) ((goal) > KMP_NOW())1297#else1298// System time is retrieved sporadically while blocking.1299extern kmp_uint64 __kmp_now_nsec();1300#define KMP_NOW() __kmp_now_nsec()1301#define KMP_BLOCKTIME_INTERVAL(team, tid) \1302 ((kmp_uint64)KMP_BLOCKTIME(team, tid) * (kmp_uint64)KMP_NSEC_PER_USEC)1303#define KMP_BLOCKING(goal, count) ((count) % 1000 != 0 || (goal) > KMP_NOW())1304#endif1305#endif // KMP_USE_MONITOR1306 1307#define KMP_MIN_STATSCOLS 401308#define KMP_MAX_STATSCOLS 40961309#define KMP_DEFAULT_STATSCOLS 801310 1311#define KMP_MIN_INTERVAL 01312#define KMP_MAX_INTERVAL (INT_MAX - 1)1313#define KMP_DEFAULT_INTERVAL 01314 1315#define KMP_MIN_CHUNK 11316#define KMP_MAX_CHUNK (INT_MAX - 1)1317#define KMP_DEFAULT_CHUNK 11318 1319#define KMP_MIN_DISP_NUM_BUFF 11320#define KMP_DFLT_DISP_NUM_BUFF 71321#define KMP_MAX_DISP_NUM_BUFF 40961322 1323#define KMP_MAX_ORDERED 81324 1325#define KMP_MAX_FIELDS 321326 1327#define KMP_MAX_BRANCH_BITS 311328 1329#define KMP_MAX_ACTIVE_LEVELS_LIMIT INT_MAX1330 1331#define KMP_MAX_DEFAULT_DEVICE_LIMIT INT_MAX1332 1333#define KMP_MAX_TASK_PRIORITY_LIMIT INT_MAX1334 1335/* Minimum number of threads before switch to TLS gtid (experimentally1336 determined) */1337/* josh TODO: what about OS X* tuning? */1338#if KMP_ARCH_X86 || KMP_ARCH_X86_641339#define KMP_TLS_GTID_MIN 51340#else1341#define KMP_TLS_GTID_MIN INT_MAX1342#endif1343 1344#define KMP_MASTER_TID(tid) (0 == (tid))1345#define KMP_WORKER_TID(tid) (0 != (tid))1346 1347#define KMP_MASTER_GTID(gtid) (0 == __kmp_tid_from_gtid((gtid)))1348#define KMP_WORKER_GTID(gtid) (0 != __kmp_tid_from_gtid((gtid)))1349#define KMP_INITIAL_GTID(gtid) (0 == (gtid))1350 1351#ifndef TRUE1352#define FALSE 01353#define TRUE (!FALSE)1354#endif1355 1356/* NOTE: all of the following constants must be even */1357 1358#if KMP_OS_WINDOWS1359#define KMP_INIT_WAIT 64U /* initial number of spin-tests */1360#define KMP_NEXT_WAIT 32U /* susequent number of spin-tests */1361#elif KMP_OS_LINUX1362#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1363#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1364#elif KMP_OS_DARWIN1365/* TODO: tune for KMP_OS_DARWIN */1366#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1367#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1368#elif KMP_OS_DRAGONFLY1369/* TODO: tune for KMP_OS_DRAGONFLY */1370#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1371#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1372#elif KMP_OS_FREEBSD1373/* TODO: tune for KMP_OS_FREEBSD */1374#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1375#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1376#elif KMP_OS_NETBSD1377/* TODO: tune for KMP_OS_NETBSD */1378#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1379#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1380#elif KMP_OS_OPENBSD1381/* TODO: tune for KMP_OS_OPENBSD */1382#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1383#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1384#elif KMP_OS_HAIKU1385/* TODO: tune for KMP_OS_HAIKU */1386#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1387#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1388#elif KMP_OS_HURD1389/* TODO: tune for KMP_OS_HURD */1390#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1391#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1392#elif KMP_OS_SOLARIS1393/* TODO: tune for KMP_OS_SOLARIS */1394#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1395#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1396#elif KMP_OS_WASI1397/* TODO: tune for KMP_OS_WASI */1398#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1399#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1400#elif KMP_OS_AIX1401/* TODO: tune for KMP_OS_AIX */1402#define KMP_INIT_WAIT 1024U /* initial number of spin-tests */1403#define KMP_NEXT_WAIT 512U /* susequent number of spin-tests */1404#endif1405 1406#if KMP_ARCH_X86 || KMP_ARCH_X86_641407typedef struct kmp_cpuid {1408 kmp_uint32 eax;1409 kmp_uint32 ebx;1410 kmp_uint32 ecx;1411 kmp_uint32 edx;1412} kmp_cpuid_t;1413 1414typedef struct kmp_cpuinfo_flags_t {1415 unsigned sse2 : 1; // 0 if SSE2 instructions are not supported, 1 otherwise.1416 unsigned rtm : 1; // 0 if RTM instructions are not supported, 1 otherwise.1417 unsigned hybrid : 1;1418 unsigned reserved : 29; // Ensure size of 32 bits1419} kmp_cpuinfo_flags_t;1420 1421typedef struct kmp_cpuinfo {1422 int initialized; // If 0, other fields are not initialized.1423 int signature; // CPUID(1).EAX1424 int family; // CPUID(1).EAX[27:20]+CPUID(1).EAX[11:8] (Extended Family+Family)1425 int model; // ( CPUID(1).EAX[19:16] << 4 ) + CPUID(1).EAX[7:4] ( ( Extended1426 // Model << 4 ) + Model)1427 int stepping; // CPUID(1).EAX[3:0] ( Stepping )1428 kmp_cpuinfo_flags_t flags;1429 int apic_id;1430 kmp_uint64 frequency; // Nominal CPU frequency in Hz.1431 char name[3 * sizeof(kmp_cpuid_t)]; // CPUID(0x80000002,0x80000003,0x80000004)1432} kmp_cpuinfo_t;1433 1434extern void __kmp_query_cpuid(kmp_cpuinfo_t *p);1435 1436#if KMP_OS_UNIX1437// subleaf is only needed for cache and topology discovery and can be set to1438// zero in most cases1439static inline void __kmp_x86_cpuid(int leaf, int subleaf, struct kmp_cpuid *p) {1440 __asm__ __volatile__("cpuid"1441 : "=a"(p->eax), "=b"(p->ebx), "=c"(p->ecx), "=d"(p->edx)1442 : "a"(leaf), "c"(subleaf));1443}1444// Load p into FPU control word1445static inline void __kmp_load_x87_fpu_control_word(const kmp_int16 *p) {1446 __asm__ __volatile__("fldcw %0" : : "m"(*p));1447}1448// Store FPU control word into p1449static inline void __kmp_store_x87_fpu_control_word(kmp_int16 *p) {1450 __asm__ __volatile__("fstcw %0" : "=m"(*p));1451}1452static inline void __kmp_clear_x87_fpu_status_word() {1453#if KMP_MIC1454 // 32-bit protected mode x87 FPU state1455 struct x87_fpu_state {1456 unsigned cw;1457 unsigned sw;1458 unsigned tw;1459 unsigned fip;1460 unsigned fips;1461 unsigned fdp;1462 unsigned fds;1463 };1464 struct x87_fpu_state fpu_state = {0, 0, 0, 0, 0, 0, 0};1465 __asm__ __volatile__("fstenv %0\n\t" // store FP env1466 "andw $0x7f00, %1\n\t" // clear 0-7,15 bits of FP SW1467 "fldenv %0\n\t" // load FP env back1468 : "+m"(fpu_state), "+m"(fpu_state.sw));1469#else1470 __asm__ __volatile__("fnclex");1471#endif // KMP_MIC1472}1473#if __SSE__1474static inline void __kmp_load_mxcsr(const kmp_uint32 *p) { _mm_setcsr(*p); }1475static inline void __kmp_store_mxcsr(kmp_uint32 *p) { *p = _mm_getcsr(); }1476#else1477static inline void __kmp_load_mxcsr(const kmp_uint32 *p) {}1478static inline void __kmp_store_mxcsr(kmp_uint32 *p) { *p = 0; }1479#endif1480#else1481// Windows still has these as external functions in assembly file1482extern void __kmp_x86_cpuid(int mode, int mode2, struct kmp_cpuid *p);1483extern void __kmp_load_x87_fpu_control_word(const kmp_int16 *p);1484extern void __kmp_store_x87_fpu_control_word(kmp_int16 *p);1485extern void __kmp_clear_x87_fpu_status_word();1486static inline void __kmp_load_mxcsr(const kmp_uint32 *p) { _mm_setcsr(*p); }1487static inline void __kmp_store_mxcsr(kmp_uint32 *p) { *p = _mm_getcsr(); }1488#endif // KMP_OS_UNIX1489 1490#define KMP_X86_MXCSR_MASK 0xffffffc0 /* ignore status flags (6 lsb) */1491 1492// User-level Monitor/Mwait1493#if KMP_HAVE_UMWAIT1494// We always try for UMWAIT first1495#if KMP_HAVE_WAITPKG_INTRINSICS1496#if KMP_HAVE_IMMINTRIN_H1497#include <immintrin.h>1498#elif KMP_HAVE_INTRIN_H1499#include <intrin.h>1500#endif1501#endif // KMP_HAVE_WAITPKG_INTRINSICS1502 1503KMP_ATTRIBUTE_TARGET_WAITPKG1504static inline int __kmp_tpause(uint32_t hint, uint64_t counter) {1505#if !KMP_HAVE_WAITPKG_INTRINSICS1506 uint32_t timeHi = uint32_t(counter >> 32);1507 uint32_t timeLo = uint32_t(counter & 0xffffffff);1508 char flag;1509 __asm__ volatile("#tpause\n.byte 0x66, 0x0F, 0xAE, 0xF1\n"1510 "setb %0"1511 // The "=q" restraint means any register accessible as rl1512 // in 32-bit mode: a, b, c, and d;1513 // in 64-bit mode: any integer register1514 : "=q"(flag)1515 : "a"(timeLo), "d"(timeHi), "c"(hint)1516 :);1517 return flag;1518#else1519 return _tpause(hint, counter);1520#endif1521}1522KMP_ATTRIBUTE_TARGET_WAITPKG1523static inline void __kmp_umonitor(void *cacheline) {1524#if !KMP_HAVE_WAITPKG_INTRINSICS1525 __asm__ volatile("# umonitor\n.byte 0xF3, 0x0F, 0xAE, 0x01 "1526 :1527 : "a"(cacheline)1528 :);1529#else1530 _umonitor(cacheline);1531#endif1532}1533KMP_ATTRIBUTE_TARGET_WAITPKG1534static inline int __kmp_umwait(uint32_t hint, uint64_t counter) {1535#if !KMP_HAVE_WAITPKG_INTRINSICS1536 uint32_t timeHi = uint32_t(counter >> 32);1537 uint32_t timeLo = uint32_t(counter & 0xffffffff);1538 char flag;1539 __asm__ volatile("#umwait\n.byte 0xF2, 0x0F, 0xAE, 0xF1\n"1540 "setb %0"1541 // The "=q" restraint means any register accessible as rl1542 // in 32-bit mode: a, b, c, and d;1543 // in 64-bit mode: any integer register1544 : "=q"(flag)1545 : "a"(timeLo), "d"(timeHi), "c"(hint)1546 :);1547 return flag;1548#else1549 return _umwait(hint, counter);1550#endif1551}1552#elif KMP_HAVE_MWAIT1553#if KMP_OS_UNIX1554#include <pmmintrin.h>1555#else1556#include <intrin.h>1557#endif1558#if KMP_OS_UNIX1559__attribute__((target("sse3")))1560#endif1561static inline void1562__kmp_mm_monitor(void *cacheline, unsigned extensions, unsigned hints) {1563 _mm_monitor(cacheline, extensions, hints);1564}1565#if KMP_OS_UNIX1566__attribute__((target("sse3")))1567#endif1568static inline void1569__kmp_mm_mwait(unsigned extensions, unsigned hints) {1570 _mm_mwait(extensions, hints);1571}1572#endif // KMP_HAVE_UMWAIT1573 1574#if KMP_ARCH_X861575extern void __kmp_x86_pause(void);1576#elif KMP_MIC1577// Performance testing on KNC (C0QS-7120 P/A/X/D, 61-core, 16 GB Memory) showed1578// regression after removal of extra PAUSE from spin loops. Changing1579// the delay from 100 to 300 showed even better performance than double PAUSE1580// on Spec OMP2001 and LCPC tasking tests, no regressions on EPCC.1581static inline void __kmp_x86_pause(void) { _mm_delay_32(300); }1582#else1583static inline void __kmp_x86_pause(void) { _mm_pause(); }1584#endif1585#define KMP_CPU_PAUSE() __kmp_x86_pause()1586#elif KMP_ARCH_PPC641587#define KMP_PPC64_PRI_LOW() __asm__ volatile("or 1, 1, 1")1588#define KMP_PPC64_PRI_MED() __asm__ volatile("or 2, 2, 2")1589#define KMP_PPC64_PRI_LOC_MB() __asm__ volatile("" : : : "memory")1590#define KMP_CPU_PAUSE() \1591 do { \1592 KMP_PPC64_PRI_LOW(); \1593 KMP_PPC64_PRI_MED(); \1594 KMP_PPC64_PRI_LOC_MB(); \1595 } while (0)1596#else1597#define KMP_CPU_PAUSE() /* nothing to do */1598#endif1599 1600#define KMP_INIT_YIELD(count) \1601 { (count) = __kmp_yield_init; }1602 1603#define KMP_INIT_BACKOFF(time) \1604 { (time) = __kmp_pause_init; }1605 1606#define KMP_OVERSUBSCRIBED \1607 (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc))1608 1609#define KMP_TRY_YIELD \1610 ((__kmp_use_yield == 1) || (__kmp_use_yield == 2 && (KMP_OVERSUBSCRIBED)))1611 1612#define KMP_TRY_YIELD_OVERSUB \1613 ((__kmp_use_yield == 1 || __kmp_use_yield == 2) && (KMP_OVERSUBSCRIBED))1614 1615#define KMP_YIELD(cond) \1616 { \1617 KMP_CPU_PAUSE(); \1618 if ((cond) && (KMP_TRY_YIELD)) \1619 __kmp_yield(); \1620 }1621 1622#define KMP_YIELD_OVERSUB() \1623 { \1624 KMP_CPU_PAUSE(); \1625 if ((KMP_TRY_YIELD_OVERSUB)) \1626 __kmp_yield(); \1627 }1628 1629// Note the decrement of 2 in the following Macros. With KMP_LIBRARY=turnaround,1630// there should be no yielding since initial value from KMP_INIT_YIELD() is odd.1631#define KMP_YIELD_SPIN(count) \1632 { \1633 KMP_CPU_PAUSE(); \1634 if (KMP_TRY_YIELD) { \1635 (count) -= 2; \1636 if (!(count)) { \1637 __kmp_yield(); \1638 (count) = __kmp_yield_next; \1639 } \1640 } \1641 }1642 1643// If TPAUSE is available & enabled, use it. If oversubscribed, use the slower1644// (C0.2) state, which improves performance of other SMT threads on the same1645// core, otherwise, use the fast (C0.1) default state, or whatever the user has1646// requested. Uses a timed TPAUSE, and exponential backoff. If TPAUSE isn't1647// available, fall back to the regular CPU pause and yield combination.1648#if KMP_HAVE_UMWAIT1649#define KMP_TPAUSE_MAX_MASK ((kmp_uint64)0xFFFF)1650#define KMP_YIELD_OVERSUB_ELSE_SPIN(count, time) \1651 { \1652 if (__kmp_tpause_enabled) { \1653 if (KMP_OVERSUBSCRIBED) { \1654 __kmp_tpause(0, (time)); \1655 } else { \1656 __kmp_tpause(__kmp_tpause_hint, (time)); \1657 } \1658 (time) = (time << 1 | 1) & KMP_TPAUSE_MAX_MASK; \1659 } else { \1660 KMP_CPU_PAUSE(); \1661 if ((KMP_TRY_YIELD_OVERSUB)) { \1662 __kmp_yield(); \1663 } else if (__kmp_use_yield == 1) { \1664 (count) -= 2; \1665 if (!(count)) { \1666 __kmp_yield(); \1667 (count) = __kmp_yield_next; \1668 } \1669 } \1670 } \1671 }1672#else1673#define KMP_YIELD_OVERSUB_ELSE_SPIN(count, time) \1674 { \1675 KMP_CPU_PAUSE(); \1676 if ((KMP_TRY_YIELD_OVERSUB)) \1677 __kmp_yield(); \1678 else if (__kmp_use_yield == 1) { \1679 (count) -= 2; \1680 if (!(count)) { \1681 __kmp_yield(); \1682 (count) = __kmp_yield_next; \1683 } \1684 } \1685 }1686#endif // KMP_HAVE_UMWAIT1687 1688/* ------------------------------------------------------------------------ */1689/* Support datatypes for the orphaned construct nesting checks. */1690/* ------------------------------------------------------------------------ */1691 1692/* When adding to this enum, add its corresponding string in cons_text_c[]1693 * array in kmp_error.cpp */1694enum cons_type {1695 ct_none,1696 ct_parallel,1697 ct_pdo,1698 ct_pdo_ordered,1699 ct_psections,1700 ct_psingle,1701 ct_critical,1702 ct_ordered_in_parallel,1703 ct_ordered_in_pdo,1704 ct_master,1705 ct_reduce,1706 ct_barrier,1707 ct_masked1708};1709 1710#define IS_CONS_TYPE_ORDERED(ct) ((ct) == ct_pdo_ordered)1711 1712struct cons_data {1713 ident_t const *ident;1714 enum cons_type type;1715 int prev;1716 kmp_user_lock_p1717 name; /* address exclusively for critical section name comparison */1718};1719 1720struct cons_header {1721 int p_top, w_top, s_top;1722 int stack_size, stack_top;1723 struct cons_data *stack_data;1724};1725 1726struct kmp_region_info {1727 char *text;1728 int offset[KMP_MAX_FIELDS];1729 int length[KMP_MAX_FIELDS];1730};1731 1732/* ---------------------------------------------------------------------- */1733/* ---------------------------------------------------------------------- */1734 1735#if KMP_OS_WINDOWS1736typedef HANDLE kmp_thread_t;1737typedef DWORD kmp_key_t;1738#endif /* KMP_OS_WINDOWS */1739 1740#if KMP_OS_UNIX1741typedef pthread_t kmp_thread_t;1742typedef pthread_key_t kmp_key_t;1743#endif1744 1745extern kmp_key_t __kmp_gtid_threadprivate_key;1746 1747typedef struct kmp_sys_info {1748 long maxrss; /* the maximum resident set size utilized (in kilobytes) */1749 long minflt; /* the number of page faults serviced without any I/O */1750 long majflt; /* the number of page faults serviced that required I/O */1751 long nswap; /* the number of times a process was "swapped" out of memory */1752 long inblock; /* the number of times the file system had to perform input */1753 long oublock; /* the number of times the file system had to perform output */1754 long nvcsw; /* the number of times a context switch was voluntarily */1755 long nivcsw; /* the number of times a context switch was forced */1756} kmp_sys_info_t;1757 1758#if USE_ITT_BUILD1759// We cannot include "kmp_itt.h" due to circular dependency. Declare the only1760// required type here. Later we will check the type meets requirements.1761typedef int kmp_itt_mark_t;1762#define KMP_ITT_DEBUG 01763#endif /* USE_ITT_BUILD */1764 1765typedef kmp_int32 kmp_critical_name[8];1766 1767/*!1768@ingroup PARALLEL1769The type for a microtask which gets passed to @ref __kmpc_fork_call().1770The arguments to the outlined function are1771@param global_tid the global thread identity of the thread executing the1772function.1773@param bound_tid the local identity of the thread executing the function1774@param ... pointers to shared variables accessed by the function.1775*/1776typedef void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid, ...);1777typedef void (*kmpc_micro_bound)(kmp_int32 *bound_tid, kmp_int32 *bound_nth,1778 ...);1779 1780/*!1781@ingroup THREADPRIVATE1782@{1783*/1784/* ---------------------------------------------------------------------------1785 */1786/* Threadprivate initialization/finalization function declarations */1787 1788/* for non-array objects: __kmpc_threadprivate_register() */1789 1790/*!1791 Pointer to the constructor function.1792 The first argument is the <tt>this</tt> pointer1793*/1794typedef void *(*kmpc_ctor)(void *);1795 1796/*!1797 Pointer to the destructor function.1798 The first argument is the <tt>this</tt> pointer1799*/1800typedef void (*kmpc_dtor)(1801 void * /*, size_t */); /* 2nd arg: magic number for KCC unused by Intel1802 compiler */1803/*!1804 Pointer to an alternate constructor.1805 The first argument is the <tt>this</tt> pointer.1806*/1807typedef void *(*kmpc_cctor)(void *, void *);1808 1809/* for array objects: __kmpc_threadprivate_register_vec() */1810/* First arg: "this" pointer */1811/* Last arg: number of array elements */1812/*!1813 Array constructor.1814 First argument is the <tt>this</tt> pointer1815 Second argument the number of array elements.1816*/1817typedef void *(*kmpc_ctor_vec)(void *, size_t);1818/*!1819 Pointer to the array destructor function.1820 The first argument is the <tt>this</tt> pointer1821 Second argument the number of array elements.1822*/1823typedef void (*kmpc_dtor_vec)(void *, size_t);1824/*!1825 Array constructor.1826 First argument is the <tt>this</tt> pointer1827 Third argument the number of array elements.1828*/1829typedef void *(*kmpc_cctor_vec)(void *, void *,1830 size_t); /* function unused by compiler */1831 1832/*!1833@}1834*/1835 1836/* keeps tracked of threadprivate cache allocations for cleanup later */1837typedef struct kmp_cached_addr {1838 void **addr; /* address of allocated cache */1839 void ***compiler_cache; /* pointer to compiler's cache */1840 void *data; /* pointer to global data */1841 struct kmp_cached_addr *next; /* pointer to next cached address */1842} kmp_cached_addr_t;1843 1844struct private_data {1845 struct private_data *next; /* The next descriptor in the list */1846 void *data; /* The data buffer for this descriptor */1847 int more; /* The repeat count for this descriptor */1848 size_t size; /* The data size for this descriptor */1849};1850 1851struct private_common {1852 struct private_common *next;1853 struct private_common *link;1854 void *gbl_addr;1855 void *par_addr; /* par_addr == gbl_addr for PRIMARY thread */1856 size_t cmn_size;1857};1858 1859struct shared_common {1860 struct shared_common *next;1861 struct private_data *pod_init;1862 void *obj_init;1863 void *gbl_addr;1864 union {1865 kmpc_ctor ctor;1866 kmpc_ctor_vec ctorv;1867 } ct;1868 union {1869 kmpc_cctor cctor;1870 kmpc_cctor_vec cctorv;1871 } cct;1872 union {1873 kmpc_dtor dtor;1874 kmpc_dtor_vec dtorv;1875 } dt;1876 size_t vec_len;1877 int is_vec;1878 size_t cmn_size;1879};1880 1881#define KMP_HASH_TABLE_LOG2 9 /* log2 of the hash table size */1882#define KMP_HASH_TABLE_SIZE \1883 (1 << KMP_HASH_TABLE_LOG2) /* size of the hash table */1884#define KMP_HASH_SHIFT 3 /* throw away this many low bits from the address */1885#define KMP_HASH(x) \1886 ((((kmp_uintptr_t)x) >> KMP_HASH_SHIFT) & (KMP_HASH_TABLE_SIZE - 1))1887 1888struct common_table {1889 struct private_common *data[KMP_HASH_TABLE_SIZE];1890};1891 1892struct shared_table {1893 struct shared_common *data[KMP_HASH_TABLE_SIZE];1894};1895 1896/* ------------------------------------------------------------------------ */1897 1898#if KMP_USE_HIER_SCHED1899// Shared barrier data that exists inside a single unit of the scheduling1900// hierarchy1901typedef struct kmp_hier_private_bdata_t {1902 kmp_int32 num_active;1903 kmp_uint64 index;1904 kmp_uint64 wait_val[2];1905} kmp_hier_private_bdata_t;1906#endif1907 1908typedef struct kmp_sched_flags {1909 unsigned ordered : 1;1910 unsigned nomerge : 1;1911 unsigned contains_last : 1;1912 unsigned use_hier : 1; // Used in KMP_USE_HIER_SCHED code1913 unsigned use_hybrid : 1; // Used in KMP_WEIGHTED_ITERATIONS_SUPPORTED code1914 unsigned unused : 27;1915} kmp_sched_flags_t;1916 1917KMP_BUILD_ASSERT(sizeof(kmp_sched_flags_t) == 4);1918 1919#if KMP_STATIC_STEAL_ENABLED1920typedef struct KMP_ALIGN_CACHE dispatch_private_info32 {1921 kmp_int32 count;1922 kmp_int32 ub;1923 /* Adding KMP_ALIGN_CACHE here doesn't help / can hurt performance */1924 kmp_int32 lb;1925 kmp_int32 st;1926 kmp_int32 tc;1927 kmp_lock_t *steal_lock; // lock used for chunk stealing1928 1929 kmp_uint32 ordered_lower;1930 kmp_uint32 ordered_upper;1931 1932 // KMP_ALIGN(32) ensures (if the KMP_ALIGN macro is turned on)1933 // a) parm3 is properly aligned and1934 // b) all parm1-4 are on the same cache line.1935 // Because of parm1-4 are used together, performance seems to be better1936 // if they are on the same cache line (not measured though).1937 1938 struct KMP_ALIGN(32) {1939 kmp_int32 parm1;1940 kmp_int32 parm2;1941 kmp_int32 parm3;1942 kmp_int32 parm4;1943 };1944 1945#if KMP_WEIGHTED_ITERATIONS_SUPPORTED1946 kmp_uint32 pchunks;1947 kmp_uint32 num_procs_with_pcore;1948 kmp_int32 first_thread_with_ecore;1949#endif1950#if KMP_OS_WINDOWS1951 kmp_int32 last_upper;1952#endif /* KMP_OS_WINDOWS */1953} dispatch_private_info32_t;1954 1955#if CACHE_LINE <= 1281956KMP_BUILD_ASSERT(sizeof(dispatch_private_info32_t) <= 128);1957#endif1958 1959typedef struct KMP_ALIGN_CACHE dispatch_private_info64 {1960 kmp_int64 count; // current chunk number for static & static-steal scheduling1961 kmp_int64 ub; /* upper-bound */1962 /* Adding KMP_ALIGN_CACHE here doesn't help / can hurt performance */1963 kmp_int64 lb; /* lower-bound */1964 kmp_int64 st; /* stride */1965 kmp_int64 tc; /* trip count (number of iterations) */1966 kmp_lock_t *steal_lock; // lock used for chunk stealing1967 1968 kmp_uint64 ordered_lower;1969 kmp_uint64 ordered_upper;1970 /* parm[1-4] are used in different ways by different scheduling algorithms */1971 1972 // KMP_ALIGN(32) ensures ( if the KMP_ALIGN macro is turned on )1973 // a) parm3 is properly aligned and1974 // b) all parm1-4 are in the same cache line.1975 // Because of parm1-4 are used together, performance seems to be better1976 // if they are in the same line (not measured though).1977 struct KMP_ALIGN(32) {1978 kmp_int64 parm1;1979 kmp_int64 parm2;1980 kmp_int64 parm3;1981 kmp_int64 parm4;1982 };1983 1984#if KMP_WEIGHTED_ITERATIONS_SUPPORTED1985 kmp_uint64 pchunks;1986 kmp_uint64 num_procs_with_pcore;1987 kmp_int64 first_thread_with_ecore;1988#endif1989 1990#if KMP_OS_WINDOWS1991 kmp_int64 last_upper;1992#endif /* KMP_OS_WINDOWS */1993} dispatch_private_info64_t;1994 1995#if CACHE_LINE <= 1281996KMP_BUILD_ASSERT(sizeof(dispatch_private_info64_t) <= 128);1997#endif1998 1999#else /* KMP_STATIC_STEAL_ENABLED */2000typedef struct KMP_ALIGN_CACHE dispatch_private_info32 {2001 kmp_int32 lb;2002 kmp_int32 ub;2003 kmp_int32 st;2004 kmp_int32 tc;2005 2006 kmp_int32 parm1;2007 kmp_int32 parm2;2008 kmp_int32 parm3;2009 kmp_int32 parm4;2010 2011 kmp_int32 count;2012 2013 kmp_uint32 ordered_lower;2014 kmp_uint32 ordered_upper;2015#if KMP_OS_WINDOWS2016 kmp_int32 last_upper;2017#endif /* KMP_OS_WINDOWS */2018} dispatch_private_info32_t;2019 2020typedef struct KMP_ALIGN_CACHE dispatch_private_info64 {2021 kmp_int64 lb; /* lower-bound */2022 kmp_int64 ub; /* upper-bound */2023 kmp_int64 st; /* stride */2024 kmp_int64 tc; /* trip count (number of iterations) */2025 2026 /* parm[1-4] are used in different ways by different scheduling algorithms */2027 kmp_int64 parm1;2028 kmp_int64 parm2;2029 kmp_int64 parm3;2030 kmp_int64 parm4;2031 2032 kmp_int64 count; /* current chunk number for static scheduling */2033 2034 kmp_uint64 ordered_lower;2035 kmp_uint64 ordered_upper;2036#if KMP_OS_WINDOWS2037 kmp_int64 last_upper;2038#endif /* KMP_OS_WINDOWS */2039} dispatch_private_info64_t;2040#endif /* KMP_STATIC_STEAL_ENABLED */2041 2042typedef struct KMP_ALIGN_CACHE dispatch_private_info {2043 union private_info {2044 dispatch_private_info32_t p32;2045 dispatch_private_info64_t p64;2046 } u;2047 enum sched_type schedule; /* scheduling algorithm */2048 kmp_sched_flags_t flags; /* flags (e.g., ordered, nomerge, etc.) */2049 std::atomic<kmp_uint32> steal_flag; // static_steal only, state of a buffer2050 kmp_int32 ordered_bumped;2051 // Stack of buffers for nest of serial regions2052 struct dispatch_private_info *next;2053 kmp_int32 type_size; /* the size of types in private_info */2054#if KMP_USE_HIER_SCHED2055 kmp_int32 hier_id;2056 void *parent; /* hierarchical scheduling parent pointer */2057#endif2058 enum cons_type pushed_ws;2059} dispatch_private_info_t;2060 2061typedef struct dispatch_shared_info32 {2062 /* chunk index under dynamic, number of idle threads under static-steal;2063 iteration index otherwise */2064 volatile kmp_uint32 iteration;2065 volatile kmp_int32 num_done;2066 volatile kmp_uint32 ordered_iteration;2067 // Dummy to retain the structure size after making ordered_iteration scalar2068 kmp_int32 ordered_dummy[KMP_MAX_ORDERED - 1];2069} dispatch_shared_info32_t;2070 2071typedef struct dispatch_shared_info64 {2072 /* chunk index under dynamic, number of idle threads under static-steal;2073 iteration index otherwise */2074 volatile kmp_uint64 iteration;2075 volatile kmp_int64 num_done;2076 volatile kmp_uint64 ordered_iteration;2077 // Dummy to retain the structure size after making ordered_iteration scalar2078 kmp_int64 ordered_dummy[KMP_MAX_ORDERED - 3];2079} dispatch_shared_info64_t;2080 2081typedef struct dispatch_shared_info {2082 union shared_info {2083 dispatch_shared_info32_t s32;2084 dispatch_shared_info64_t s64;2085 } u;2086 volatile kmp_uint32 buffer_index;2087 volatile kmp_int32 doacross_buf_idx; // teamwise index2088 volatile kmp_uint32 *doacross_flags; // shared array of iteration flags (0/1)2089 kmp_int32 doacross_num_done; // count finished threads2090#if KMP_USE_HIER_SCHED2091 void *hier;2092#endif2093#if KMP_HWLOC_ENABLED2094 // When linking with libhwloc, the ORDERED EPCC test slows down on big2095 // machines (> 48 cores). Performance analysis showed that a cache thrash2096 // was occurring and this padding helps alleviate the problem.2097 char padding[64];2098#endif // KMP_HWLOC_ENABLED2099} dispatch_shared_info_t;2100 2101typedef struct kmp_disp {2102 /* Vector for ORDERED SECTION */2103 void (*th_deo_fcn)(int *gtid, int *cid, ident_t *);2104 /* Vector for END ORDERED SECTION */2105 void (*th_dxo_fcn)(int *gtid, int *cid, ident_t *);2106 2107 dispatch_shared_info_t *th_dispatch_sh_current;2108 dispatch_private_info_t *th_dispatch_pr_current;2109 2110 dispatch_private_info_t *th_disp_buffer;2111 kmp_uint32 th_disp_index;2112 kmp_int32 th_doacross_buf_idx; // thread's doacross buffer index2113 volatile kmp_uint32 *th_doacross_flags; // pointer to shared array of flags2114 kmp_int64 *th_doacross_info; // info on loop bounds2115#if KMP_USE_INTERNODE_ALIGNMENT2116 char more_padding[INTERNODE_CACHE_LINE];2117#endif2118} kmp_disp_t;2119 2120/* ------------------------------------------------------------------------ */2121/* Barrier stuff */2122 2123/* constants for barrier state update */2124#define KMP_INIT_BARRIER_STATE 0 /* should probably start from zero */2125#define KMP_BARRIER_SLEEP_BIT 0 /* bit used for suspend/sleep part of state */2126#define KMP_BARRIER_UNUSED_BIT 1 // bit that must never be set for valid state2127#define KMP_BARRIER_BUMP_BIT 2 /* lsb used for bump of go/arrived state */2128 2129#define KMP_BARRIER_SLEEP_STATE (1 << KMP_BARRIER_SLEEP_BIT)2130#define KMP_BARRIER_UNUSED_STATE (1 << KMP_BARRIER_UNUSED_BIT)2131#define KMP_BARRIER_STATE_BUMP (1 << KMP_BARRIER_BUMP_BIT)2132 2133#if (KMP_BARRIER_SLEEP_BIT >= KMP_BARRIER_BUMP_BIT)2134#error "Barrier sleep bit must be smaller than barrier bump bit"2135#endif2136#if (KMP_BARRIER_UNUSED_BIT >= KMP_BARRIER_BUMP_BIT)2137#error "Barrier unused bit must be smaller than barrier bump bit"2138#endif2139 2140// Constants for release barrier wait state: currently, hierarchical only2141#define KMP_BARRIER_NOT_WAITING 0 // Normal state; worker not in wait_sleep2142#define KMP_BARRIER_OWN_FLAG \2143 1 // Normal state; worker waiting on own b_go flag in release2144#define KMP_BARRIER_PARENT_FLAG \2145 2 // Special state; worker waiting on parent's b_go flag in release2146#define KMP_BARRIER_SWITCH_TO_OWN_FLAG \2147 3 // Special state; tells worker to shift from parent to own b_go2148#define KMP_BARRIER_SWITCHING \2149 4 // Special state; worker resets appropriate flag on wake-up2150 2151#define KMP_NOT_SAFE_TO_REAP \2152 0 // Thread th_reap_state: not safe to reap (tasking)2153#define KMP_SAFE_TO_REAP 1 // Thread th_reap_state: safe to reap (not tasking)2154 2155// The flag_type describes the storage used for the flag.2156enum flag_type {2157 flag32, /**< atomic 32 bit flags */2158 flag64, /**< 64 bit flags */2159 atomic_flag64, /**< atomic 64 bit flags */2160 flag_oncore, /**< special 64-bit flag for on-core barrier (hierarchical) */2161 flag_unset2162};2163 2164enum barrier_type {2165 bs_plain_barrier = 0, /* 0, All non-fork/join barriers (except reduction2166 barriers if enabled) */2167 bs_forkjoin_barrier, /* 1, All fork/join (parallel region) barriers */2168#if KMP_FAST_REDUCTION_BARRIER2169 bs_reduction_barrier, /* 2, All barriers that are used in reduction */2170#endif // KMP_FAST_REDUCTION_BARRIER2171 bs_last_barrier /* Just a placeholder to mark the end */2172};2173 2174// to work with reduction barriers just like with plain barriers2175#if !KMP_FAST_REDUCTION_BARRIER2176#define bs_reduction_barrier bs_plain_barrier2177#endif // KMP_FAST_REDUCTION_BARRIER2178 2179typedef enum kmp_bar_pat { /* Barrier communication patterns */2180 bp_linear_bar =2181 0, /* Single level (degenerate) tree */2182 bp_tree_bar =2183 1, /* Balanced tree with branching factor 2^n */2184 bp_hyper_bar = 2, /* Hypercube-embedded tree with min2185 branching factor 2^n */2186 bp_hierarchical_bar = 3, /* Machine hierarchy tree */2187 bp_dist_bar = 4, /* Distributed barrier */2188 bp_last_bar /* Placeholder to mark the end */2189} kmp_bar_pat_e;2190 2191#define KMP_BARRIER_ICV_PUSH 12192 2193/* Record for holding the values of the internal controls stack records */2194typedef struct kmp_internal_control {2195 int serial_nesting_level; /* corresponds to the value of the2196 th_team_serialized field */2197 kmp_int8 dynamic; /* internal control for dynamic adjustment of threads (per2198 thread) */2199 kmp_int82200 bt_set; /* internal control for whether blocktime is explicitly set */2201 int blocktime; /* internal control for blocktime */2202#if KMP_USE_MONITOR2203 int bt_intervals; /* internal control for blocktime intervals */2204#endif2205 int nproc; /* internal control for #threads for next parallel region (per2206 thread) */2207 int thread_limit; /* internal control for thread-limit-var */2208 int task_thread_limit; /* internal control for thread-limit-var of a task*/2209 int max_active_levels; /* internal control for max_active_levels */2210 kmp_r_sched_t2211 sched; /* internal control for runtime schedule {sched,chunk} pair */2212 kmp_proc_bind_t proc_bind; /* internal control for affinity */2213 kmp_int32 default_device; /* internal control for default device */2214 struct kmp_internal_control *next;2215} kmp_internal_control_t;2216 2217static inline void copy_icvs(kmp_internal_control_t *dst,2218 kmp_internal_control_t *src) {2219 *dst = *src;2220}2221 2222/* Thread barrier needs volatile barrier fields */2223typedef struct KMP_ALIGN_CACHE kmp_bstate {2224 // th_fixed_icvs is aligned by virtue of kmp_bstate being aligned (and all2225 // uses of it). It is not explicitly aligned below, because we *don't* want2226 // it to be padded -- instead, we fit b_go into the same cache line with2227 // th_fixed_icvs, enabling NGO cache lines stores in the hierarchical barrier.2228 kmp_internal_control_t th_fixed_icvs; // Initial ICVs for the thread2229 // Tuck b_go into end of th_fixed_icvs cache line, so it can be stored with2230 // same NGO store2231 volatile kmp_uint64 b_go; // STATE => task should proceed (hierarchical)2232 KMP_ALIGN_CACHE volatile kmp_uint642233 b_arrived; // STATE => task reached synch point.2234 kmp_uint32 *skip_per_level;2235 kmp_uint32 my_level;2236 kmp_int32 parent_tid;2237 kmp_int32 old_tid;2238 kmp_uint32 depth;2239 struct kmp_bstate *parent_bar;2240 kmp_team_t *team;2241 kmp_uint64 leaf_state;2242 kmp_uint32 nproc;2243 kmp_uint8 base_leaf_kids;2244 kmp_uint8 leaf_kids;2245 kmp_uint8 offset;2246 kmp_uint8 wait_flag;2247 kmp_uint8 use_oncore_barrier;2248#if USE_DEBUGGER2249 // The following field is intended for the debugger solely. Only the worker2250 // thread itself accesses this field: the worker increases it by 1 when it2251 // arrives to a barrier.2252 KMP_ALIGN_CACHE kmp_uint b_worker_arrived;2253#endif /* USE_DEBUGGER */2254} kmp_bstate_t;2255 2256union KMP_ALIGN_CACHE kmp_barrier_union {2257 double b_align; /* use worst case alignment */2258 char b_pad[KMP_PAD(kmp_bstate_t, CACHE_LINE)];2259 kmp_bstate_t bb;2260};2261 2262typedef union kmp_barrier_union kmp_balign_t;2263 2264/* Team barrier needs only non-volatile arrived counter */2265union KMP_ALIGN_CACHE kmp_barrier_team_union {2266 double b_align; /* use worst case alignment */2267 char b_pad[CACHE_LINE];2268 struct {2269 kmp_uint64 b_arrived; /* STATE => task reached synch point. */2270#if USE_DEBUGGER2271 // The following two fields are indended for the debugger solely. Only2272 // primary thread of the team accesses these fields: the first one is2273 // increased by 1 when the primary thread arrives to a barrier, the second2274 // one is increased by one when all the threads arrived.2275 kmp_uint b_master_arrived;2276 kmp_uint b_team_arrived;2277#endif2278 };2279};2280 2281typedef union kmp_barrier_team_union kmp_balign_team_t;2282 2283/* Padding for Linux* OS pthreads condition variables and mutexes used to signal2284 threads when a condition changes. This is to workaround an NPTL bug where2285 padding was added to pthread_cond_t which caused the initialization routine2286 to write outside of the structure if compiled on pre-NPTL threads. */2287#if KMP_OS_WINDOWS2288typedef struct kmp_win32_mutex {2289 /* The Lock */2290 CRITICAL_SECTION cs;2291} kmp_win32_mutex_t;2292 2293typedef struct kmp_win32_cond {2294 /* Count of the number of waiters. */2295 int waiters_count_;2296 2297 /* Serialize access to <waiters_count_> */2298 kmp_win32_mutex_t waiters_count_lock_;2299 2300 /* Number of threads to release via a <cond_broadcast> or a <cond_signal> */2301 int release_count_;2302 2303 /* Keeps track of the current "generation" so that we don't allow */2304 /* one thread to steal all the "releases" from the broadcast. */2305 int wait_generation_count_;2306 2307 /* A manual-reset event that's used to block and release waiting threads. */2308 HANDLE event_;2309} kmp_win32_cond_t;2310#endif2311 2312#if KMP_OS_UNIX2313 2314union KMP_ALIGN_CACHE kmp_cond_union {2315 double c_align;2316 char c_pad[CACHE_LINE];2317 pthread_cond_t c_cond;2318};2319 2320typedef union kmp_cond_union kmp_cond_align_t;2321 2322union KMP_ALIGN_CACHE kmp_mutex_union {2323 double m_align;2324 char m_pad[CACHE_LINE];2325 pthread_mutex_t m_mutex;2326};2327 2328typedef union kmp_mutex_union kmp_mutex_align_t;2329 2330#endif /* KMP_OS_UNIX */2331 2332typedef struct kmp_desc_base {2333 void *ds_stackbase;2334 size_t ds_stacksize;2335 int ds_stackgrow;2336 kmp_thread_t ds_thread;2337 volatile int ds_tid;2338 int ds_gtid;2339#if KMP_OS_WINDOWS2340 volatile int ds_alive;2341 DWORD ds_thread_id;2342/* ds_thread keeps thread handle on Windows* OS. It is enough for RTL purposes.2343 However, debugger support (libomp_db) cannot work with handles, because they2344 uncomparable. For example, debugger requests info about thread with handle h.2345 h is valid within debugger process, and meaningless within debugee process.2346 Even if h is duped by call to DuplicateHandle(), so the result h' is valid2347 within debugee process, but it is a *new* handle which does *not* equal to2348 any other handle in debugee... The only way to compare handles is convert2349 them to system-wide ids. GetThreadId() function is available only in2350 Longhorn and Server 2003. :-( In contrast, GetCurrentThreadId() is available2351 on all Windows* OS flavours (including Windows* 95). Thus, we have to get2352 thread id by call to GetCurrentThreadId() from within the thread and save it2353 to let libomp_db identify threads. */2354#endif /* KMP_OS_WINDOWS */2355} kmp_desc_base_t;2356 2357typedef union KMP_ALIGN_CACHE kmp_desc {2358 double ds_align; /* use worst case alignment */2359 char ds_pad[KMP_PAD(kmp_desc_base_t, CACHE_LINE)];2360 kmp_desc_base_t ds;2361} kmp_desc_t;2362 2363typedef struct kmp_local {2364 volatile int this_construct; /* count of single's encountered by thread */2365 void *reduce_data;2366#if KMP_USE_BGET2367 void *bget_data;2368 void *bget_list;2369#if !USE_CMP_XCHG_FOR_BGET2370#ifdef USE_QUEUING_LOCK_FOR_BGET2371 kmp_lock_t bget_lock; /* Lock for accessing bget free list */2372#else2373 kmp_bootstrap_lock_t bget_lock; // Lock for accessing bget free list. Must be2374// bootstrap lock so we can use it at library2375// shutdown.2376#endif /* USE_LOCK_FOR_BGET */2377#endif /* ! USE_CMP_XCHG_FOR_BGET */2378#endif /* KMP_USE_BGET */2379 2380 PACKED_REDUCTION_METHOD_T2381 packed_reduction_method; /* stored by __kmpc_reduce*(), used by2382 __kmpc_end_reduce*() */2383 2384} kmp_local_t;2385 2386#define KMP_CHECK_UPDATE(a, b) \2387 if ((a) != (b)) \2388 (a) = (b)2389#define KMP_CHECK_UPDATE_SYNC(a, b) \2390 if ((a) != (b)) \2391 TCW_SYNC_PTR((a), (b))2392 2393#define get__blocktime(xteam, xtid) \2394 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime)2395#define get__bt_set(xteam, xtid) \2396 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set)2397#if KMP_USE_MONITOR2398#define get__bt_intervals(xteam, xtid) \2399 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals)2400#endif2401 2402#define get__dynamic_2(xteam, xtid) \2403 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.dynamic)2404#define get__nproc_2(xteam, xtid) \2405 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.nproc)2406#define get__sched_2(xteam, xtid) \2407 ((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.sched)2408 2409#define set__blocktime_team(xteam, xtid, xval) \2410 (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.blocktime) = \2411 (xval))2412 2413#if KMP_USE_MONITOR2414#define set__bt_intervals_team(xteam, xtid, xval) \2415 (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_intervals) = \2416 (xval))2417#endif2418 2419#define set__bt_set_team(xteam, xtid, xval) \2420 (((xteam)->t.t_threads[(xtid)]->th.th_current_task->td_icvs.bt_set) = (xval))2421 2422#define set__dynamic(xthread, xval) \2423 (((xthread)->th.th_current_task->td_icvs.dynamic) = (xval))2424#define get__dynamic(xthread) \2425 (((xthread)->th.th_current_task->td_icvs.dynamic) ? (FTN_TRUE) : (FTN_FALSE))2426 2427#define set__nproc(xthread, xval) \2428 (((xthread)->th.th_current_task->td_icvs.nproc) = (xval))2429 2430#define set__thread_limit(xthread, xval) \2431 (((xthread)->th.th_current_task->td_icvs.thread_limit) = (xval))2432 2433#define set__max_active_levels(xthread, xval) \2434 (((xthread)->th.th_current_task->td_icvs.max_active_levels) = (xval))2435 2436#define get__max_active_levels(xthread) \2437 ((xthread)->th.th_current_task->td_icvs.max_active_levels)2438 2439#define set__sched(xthread, xval) \2440 (((xthread)->th.th_current_task->td_icvs.sched) = (xval))2441 2442#define set__proc_bind(xthread, xval) \2443 (((xthread)->th.th_current_task->td_icvs.proc_bind) = (xval))2444#define get__proc_bind(xthread) \2445 ((xthread)->th.th_current_task->td_icvs.proc_bind)2446 2447// OpenMP tasking data structures2448 2449typedef enum kmp_tasking_mode {2450 tskm_immediate_exec = 0,2451 tskm_extra_barrier = 1,2452 tskm_task_teams = 2,2453 tskm_max = 22454} kmp_tasking_mode_t;2455 2456extern kmp_tasking_mode_t2457 __kmp_tasking_mode; /* determines how/when to execute tasks */2458extern int __kmp_task_stealing_constraint;2459extern int __kmp_enable_task_throttling;2460extern kmp_int32 __kmp_default_device; // Set via OMP_DEFAULT_DEVICE if2461// specified, defaults to 0 otherwise2462// Set via OMP_MAX_TASK_PRIORITY if specified, defaults to 0 otherwise2463extern kmp_int32 __kmp_max_task_priority;2464// Set via KMP_TASKLOOP_MIN_TASKS if specified, defaults to 0 otherwise2465extern kmp_uint64 __kmp_taskloop_min_tasks;2466 2467/* NOTE: kmp_taskdata_t and kmp_task_t structures allocated in single block with2468 taskdata first */2469#define KMP_TASK_TO_TASKDATA(task) (((kmp_taskdata_t *)task) - 1)2470#define KMP_TASKDATA_TO_TASK(taskdata) (kmp_task_t *)(taskdata + 1)2471 2472// The tt_found_tasks flag is a signal to all threads in the team that tasks2473// were spawned and queued since the previous barrier release.2474#define KMP_TASKING_ENABLED(task_team) \2475 (TRUE == TCR_SYNC_4((task_team)->tt.tt_found_tasks))2476/*!2477@ingroup BASIC_TYPES2478@{2479*/2480 2481/*!2482 */2483typedef kmp_int32 (*kmp_routine_entry_t)(kmp_int32, void *);2484 2485typedef union kmp_cmplrdata {2486 kmp_int32 priority; /**< priority specified by user for the task */2487 kmp_routine_entry_t2488 destructors; /* pointer to function to invoke deconstructors of2489 firstprivate C++ objects */2490 /* future data */2491} kmp_cmplrdata_t;2492 2493/* sizeof_kmp_task_t passed as arg to kmpc_omp_task call */2494/*!2495 */2496typedef struct kmp_task { /* GEH: Shouldn't this be aligned somehow? */2497 void *shareds; /**< pointer to block of pointers to shared vars */2498 kmp_routine_entry_t2499 routine; /**< pointer to routine to call for executing task */2500 kmp_int32 part_id; /**< part id for the task */2501 kmp_cmplrdata_t2502 data1; /* Two known optional additions: destructors and priority */2503 kmp_cmplrdata_t data2; /* Process destructors first, priority second */2504 /* future data */2505 /* private vars */2506} kmp_task_t;2507 2508/*!2509@}2510*/2511 2512typedef struct kmp_taskgroup {2513 std::atomic<kmp_int32> count; // number of allocated and incomplete tasks2514 std::atomic<kmp_int32>2515 cancel_request; // request for cancellation of this taskgroup2516 struct kmp_taskgroup *parent; // parent taskgroup2517 // Block of data to perform task reduction2518 void *reduce_data; // reduction related info2519 kmp_int32 reduce_num_data; // number of data items to reduce2520 uintptr_t *gomp_data; // gomp reduction data2521} kmp_taskgroup_t;2522 2523// forward declarations2524typedef union kmp_depnode kmp_depnode_t;2525typedef struct kmp_depnode_list kmp_depnode_list_t;2526typedef struct kmp_dephash_entry kmp_dephash_entry_t;2527 2528// macros for checking dep flag as an integer2529#define KMP_DEP_IN 0x12530#define KMP_DEP_OUT 0x22531#define KMP_DEP_INOUT 0x32532#define KMP_DEP_MTX 0x42533#define KMP_DEP_SET 0x82534#define KMP_DEP_ALL 0x802535// Compiler sends us this info. Note: some test cases contain an explicit copy2536// of this struct and should be in sync with any changes here.2537typedef struct kmp_depend_info {2538 kmp_intptr_t base_addr;2539 size_t len;2540 union {2541 kmp_uint8 flag; // flag as an unsigned char2542 struct { // flag as a set of 8 bits2543#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)2544 /* Same fields as in the #else branch, but in reverse order */2545 unsigned all : 1;2546 unsigned unused : 3;2547 unsigned set : 1;2548 unsigned mtx : 1;2549 unsigned out : 1;2550 unsigned in : 1;2551#else2552 unsigned in : 1;2553 unsigned out : 1;2554 unsigned mtx : 1;2555 unsigned set : 1;2556 unsigned unused : 3;2557 unsigned all : 1;2558#endif2559 } flags;2560 };2561} kmp_depend_info_t;2562 2563// Internal structures to work with task dependencies:2564struct kmp_depnode_list {2565 kmp_depnode_t *node;2566 kmp_depnode_list_t *next;2567};2568 2569// Max number of mutexinoutset dependencies per node2570#define MAX_MTX_DEPS 42571 2572typedef struct kmp_base_depnode {2573 kmp_depnode_list_t *successors; /* used under lock */2574 kmp_task_t *task; /* non-NULL if depnode is active, used under lock */2575 kmp_lock_t *mtx_locks[MAX_MTX_DEPS]; /* lock mutexinoutset dependent tasks */2576 kmp_int32 mtx_num_locks; /* number of locks in mtx_locks array */2577 kmp_lock_t lock; /* guards shared fields: task, successors */2578#if KMP_SUPPORT_GRAPH_OUTPUT2579 kmp_uint32 id;2580#endif2581 std::atomic<kmp_int32> npredecessors;2582 std::atomic<kmp_int32> nrefs;2583} kmp_base_depnode_t;2584 2585union KMP_ALIGN_CACHE kmp_depnode {2586 double dn_align; /* use worst case alignment */2587 char dn_pad[KMP_PAD(kmp_base_depnode_t, CACHE_LINE)];2588 kmp_base_depnode_t dn;2589};2590 2591struct kmp_dephash_entry {2592 kmp_intptr_t addr;2593 kmp_depnode_t *last_out;2594 kmp_depnode_list_t *last_set;2595 kmp_depnode_list_t *prev_set;2596 kmp_uint8 last_flag;2597 kmp_lock_t *mtx_lock; /* is referenced by depnodes w/mutexinoutset dep */2598 kmp_dephash_entry_t *next_in_bucket;2599};2600 2601typedef struct kmp_dephash {2602 kmp_dephash_entry_t **buckets;2603 size_t size;2604 kmp_depnode_t *last_all;2605 size_t generation;2606 kmp_uint32 nelements;2607 kmp_uint32 nconflicts;2608} kmp_dephash_t;2609 2610typedef struct kmp_task_affinity_info {2611 kmp_intptr_t base_addr;2612 size_t len;2613 struct {2614 bool flag1 : 1;2615 bool flag2 : 1;2616 kmp_int32 reserved : 30;2617 } flags;2618} kmp_task_affinity_info_t;2619 2620typedef enum kmp_event_type_t {2621 KMP_EVENT_UNINITIALIZED = 0,2622 KMP_EVENT_ALLOW_COMPLETION = 12623} kmp_event_type_t;2624 2625typedef struct {2626 kmp_event_type_t type;2627 kmp_tas_lock_t lock;2628 union {2629 kmp_task_t *task;2630 } ed;2631} kmp_event_t;2632 2633#if OMPX_TASKGRAPH2634// Initial number of allocated nodes while recording2635#define INIT_MAPSIZE 502636 2637typedef struct kmp_taskgraph_flags { /*This needs to be exactly 32 bits */2638 unsigned nowait : 1;2639 unsigned re_record : 1;2640 unsigned reserved : 30;2641} kmp_taskgraph_flags_t;2642 2643/// Represents a TDG node2644typedef struct kmp_node_info {2645 kmp_task_t *task; // Pointer to the actual task2646 kmp_int32 *successors; // Array of the succesors ids2647 kmp_int32 nsuccessors; // Number of succesors of the node2648 std::atomic<kmp_int32>2649 npredecessors_counter; // Number of predessors on the fly2650 kmp_int32 npredecessors; // Total number of predecessors2651 kmp_int32 successors_size; // Number of allocated succesors ids2652 kmp_taskdata_t *parent_task; // Parent implicit task2653} kmp_node_info_t;2654 2655/// Represent a TDG's current status2656typedef enum kmp_tdg_status {2657 KMP_TDG_NONE = 0,2658 KMP_TDG_RECORDING = 1,2659 KMP_TDG_READY = 22660} kmp_tdg_status_t;2661 2662/// Structure that contains a TDG2663typedef struct kmp_tdg_info {2664 kmp_int32 tdg_id; // Unique idenfifier of the TDG2665 kmp_taskgraph_flags_t tdg_flags; // Flags related to a TDG2666 kmp_int32 map_size; // Number of allocated TDG nodes2667 kmp_int32 num_roots; // Number of roots tasks int the TDG2668 kmp_int32 *root_tasks; // Array of tasks identifiers that are roots2669 kmp_node_info_t *record_map; // Array of TDG nodes2670 kmp_tdg_status_t tdg_status =2671 KMP_TDG_NONE; // Status of the TDG (recording, ready...)2672 std::atomic<kmp_int32> num_tasks; // Number of TDG nodes2673 kmp_bootstrap_lock_t2674 graph_lock; // Protect graph attributes when updated via taskloop_recur2675 // Taskloop reduction related2676 void *rec_taskred_data; // Data to pass to __kmpc_task_reduction_init or2677 // __kmpc_taskred_init2678 kmp_int32 rec_num_taskred;2679} kmp_tdg_info_t;2680 2681extern int __kmp_tdg_dot;2682extern kmp_int32 __kmp_max_tdgs;2683extern kmp_tdg_info_t **__kmp_global_tdgs;2684extern kmp_int32 __kmp_curr_tdg_idx;2685extern kmp_int32 __kmp_successors_size;2686extern std::atomic<kmp_int32> __kmp_tdg_task_id;2687extern kmp_int32 __kmp_num_tdg;2688#endif2689 2690typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */2691#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)2692 /* Same fields as in the #else branch, but in reverse order */2693#if OMPX_TASKGRAPH2694 unsigned reserved31 : 4;2695 unsigned onced : 1;2696#else2697 unsigned reserved31 : 5;2698#endif2699 unsigned hidden_helper : 1;2700 unsigned target : 1;2701 unsigned native : 1;2702 unsigned freed : 1;2703 unsigned complete : 1;2704 unsigned executing : 1;2705 unsigned started : 1;2706 unsigned team_serial : 1;2707 unsigned tasking_ser : 1;2708 unsigned task_serial : 1;2709 unsigned tasktype : 1;2710 unsigned reserved : 7;2711 unsigned transparent : 1;2712 unsigned free_agent_eligible : 1;2713 unsigned detachable : 1;2714 unsigned priority_specified : 1;2715 unsigned proxy : 1;2716 unsigned destructors_thunk : 1;2717 unsigned merged_if0 : 1;2718 unsigned final : 1;2719 unsigned tiedness : 1;2720#else2721 /* Compiler flags */ /* Total compiler flags must be 16 bits */2722 unsigned tiedness : 1; /* task is either tied (1) or untied (0) */2723 unsigned final : 1; /* task is final(1) so execute immediately */2724 unsigned merged_if0 : 1; /* no __kmpc_task_{begin/complete}_if0 calls in if02725 code path */2726 unsigned destructors_thunk : 1; /* set if the compiler creates a thunk to2727 invoke destructors from the runtime */2728 unsigned proxy : 1; /* task is a proxy task (it will be executed outside the2729 context of the RTL) */2730 unsigned priority_specified : 1; /* set if the compiler provides priority2731 setting for the task */2732 unsigned detachable : 1; /* 1 == can detach */2733 unsigned free_agent_eligible : 1; /* set if task can be executed by a2734 free-agent thread */2735 unsigned transparent : 1; /* transparent task support (compiler uses this) */2736 unsigned reserved : 7; /* reserved for compiler use */2737 2738 /* Library flags */ /* Total library flags must be 16 bits */2739 unsigned tasktype : 1; /* task is either explicit(1) or implicit (0) */2740 unsigned task_serial : 1; // task is executed immediately (1) or deferred (0)2741 unsigned tasking_ser : 1; // all tasks in team are either executed immediately2742 // (1) or may be deferred (0)2743 unsigned team_serial : 1; // entire team is serial (1) [1 thread] or parallel2744 // (0) [>= 2 threads]2745 /* If either team_serial or tasking_ser is set, task team may be NULL */2746 /* Task State Flags: */2747 unsigned started : 1; /* 1==started, 0==not started */2748 unsigned executing : 1; /* 1==executing, 0==not executing */2749 unsigned complete : 1; /* 1==complete, 0==not complete */2750 unsigned freed : 1; /* 1==freed, 0==allocated */2751 unsigned native : 1; /* 1==gcc-compiled task, 0==intel */2752 unsigned target : 1;2753 unsigned hidden_helper : 1; /* 1 == hidden helper task */2754#if OMPX_TASKGRAPH2755 unsigned onced : 1; /* 1==ran once already, 0==never ran, record & replay purposes */2756 unsigned reserved31 : 4; /* reserved for library use */2757#else2758 unsigned reserved31 : 5; /* reserved for library use */2759#endif2760#endif2761} kmp_tasking_flags_t;2762 2763typedef struct kmp_target_data {2764 void *async_handle; // libomptarget async handle for task completion query2765} kmp_target_data_t;2766 2767struct kmp_taskdata { /* aligned during dynamic allocation */2768 kmp_int32 td_task_id; /* id, assigned by debugger */2769 kmp_tasking_flags_t td_flags; /* task flags */2770 kmp_team_t *td_team; /* team for this task */2771 kmp_info_p *td_alloc_thread; /* thread that allocated data structures */2772 /* Currently not used except for perhaps IDB */2773 kmp_taskdata_t *td_parent; /* parent task */2774 kmp_int32 td_level; /* task nesting level */2775 std::atomic<kmp_int32> td_untied_count; // untied task active parts counter2776 ident_t *td_ident; /* task identifier */2777 // Taskwait data.2778 ident_t *td_taskwait_ident;2779 kmp_uint32 td_taskwait_counter;2780 kmp_int32 td_taskwait_thread; /* gtid + 1 of thread encountered taskwait */2781 KMP_ALIGN_CACHE kmp_internal_control_t2782 td_icvs; /* Internal control variables for the task */2783 KMP_ALIGN_CACHE std::atomic<kmp_int32>2784 td_allocated_child_tasks; /* Child tasks (+ current task) not yet2785 deallocated */2786 std::atomic<kmp_int32>2787 td_incomplete_child_tasks; /* Child tasks not yet complete */2788 kmp_taskgroup_t2789 *td_taskgroup; // Each task keeps pointer to its current taskgroup2790 kmp_dephash_t2791 *td_dephash; // Dependencies for children tasks are tracked from here2792 kmp_depnode_t2793 *td_depnode; // Pointer to graph node if this task has dependencies2794 kmp_task_team_t *td_task_team;2795 size_t td_size_alloc; // Size of task structure, including shareds etc.2796#if defined(KMP_GOMP_COMPAT)2797 // 4 or 8 byte integers for the loop bounds in GOMP_taskloop2798 kmp_int32 td_size_loop_bounds;2799#endif2800 kmp_taskdata_t *td_last_tied; // keep tied task for task scheduling constraint2801#if defined(KMP_GOMP_COMPAT)2802 // GOMP sends in a copy function for copy constructors2803 void (*td_copy_func)(void *, void *);2804#endif2805 kmp_event_t td_allow_completion_event;2806#if OMPT_SUPPORT2807 ompt_task_info_t ompt_task_info;2808#endif2809#if OMPX_TASKGRAPH2810 bool is_taskgraph = 0; // whether the task is within a TDG2811 kmp_tdg_info_t *tdg; // used to associate task with a TDG2812 kmp_int32 td_tdg_task_id; // local task id in its TDG2813#endif2814 kmp_target_data_t td_target_data;2815}; // struct kmp_taskdata2816 2817// Make sure padding above worked2818KMP_BUILD_ASSERT(sizeof(kmp_taskdata_t) % sizeof(void *) == 0);2819 2820// Data for task team but per thread2821typedef struct kmp_base_thread_data {2822 kmp_info_p *td_thr; // Pointer back to thread info2823 // Used only in __kmp_execute_tasks_template, maybe not avail until task is2824 // queued?2825 kmp_bootstrap_lock_t td_deque_lock; // Lock for accessing deque2826 kmp_taskdata_t *2827 *td_deque; // Deque of tasks encountered by td_thr, dynamically allocated2828 kmp_int32 td_deque_size; // Size of deck2829 kmp_uint32 td_deque_head; // Head of deque (will wrap)2830 kmp_uint32 td_deque_tail; // Tail of deque (will wrap)2831 kmp_int32 td_deque_ntasks; // Number of tasks in deque2832 // GEH: shouldn't this be volatile since used in while-spin?2833 kmp_int32 td_deque_last_stolen; // Thread number of last successful steal2834} kmp_base_thread_data_t;2835 2836#define TASK_DEQUE_BITS 8 // Used solely to define INITIAL_TASK_DEQUE_SIZE2837#define INITIAL_TASK_DEQUE_SIZE (1 << TASK_DEQUE_BITS)2838 2839#define TASK_DEQUE_SIZE(td) ((td).td_deque_size)2840#define TASK_DEQUE_MASK(td) ((td).td_deque_size - 1)2841 2842typedef union KMP_ALIGN_CACHE kmp_thread_data {2843 kmp_base_thread_data_t td;2844 double td_align; /* use worst case alignment */2845 char td_pad[KMP_PAD(kmp_base_thread_data_t, CACHE_LINE)];2846} kmp_thread_data_t;2847 2848typedef struct kmp_task_pri {2849 kmp_thread_data_t td;2850 kmp_int32 priority;2851 kmp_task_pri *next;2852} kmp_task_pri_t;2853 2854// Data for task teams which are used when tasking is enabled for the team2855typedef struct kmp_base_task_team {2856 kmp_bootstrap_lock_t2857 tt_threads_lock; /* Lock used to allocate per-thread part of task team */2858 /* must be bootstrap lock since used at library shutdown*/2859 2860 // TODO: check performance vs kmp_tas_lock_t2861 kmp_bootstrap_lock_t tt_task_pri_lock; /* Lock to access priority tasks */2862 kmp_task_pri_t *tt_task_pri_list;2863 2864 kmp_task_team_t *tt_next; /* For linking the task team free list */2865 kmp_thread_data_t2866 *tt_threads_data; /* Array of per-thread structures for task team */2867 /* Data survives task team deallocation */2868 kmp_int32 tt_found_tasks; /* Have we found tasks and queued them while2869 executing this team? */2870 /* TRUE means tt_threads_data is set up and initialized */2871 kmp_int32 tt_nproc; /* #threads in team */2872 kmp_int32 tt_max_threads; // # entries allocated for threads_data array2873 kmp_int32 tt_found_proxy_tasks; // found proxy tasks since last barrier2874 kmp_int32 tt_untied_task_encountered;2875 std::atomic<kmp_int32> tt_num_task_pri; // number of priority tasks enqueued2876 // There is hidden helper thread encountered in this task team so that we must2877 // wait when waiting on task team2878 kmp_int32 tt_hidden_helper_task_encountered;2879 2880 KMP_ALIGN_CACHE2881 std::atomic<kmp_int32> tt_unfinished_threads; /* #threads still active */2882 2883 KMP_ALIGN_CACHE2884 volatile kmp_uint322885 tt_active; /* is the team still actively executing tasks */2886} kmp_base_task_team_t;2887 2888union KMP_ALIGN_CACHE kmp_task_team {2889 kmp_base_task_team_t tt;2890 double tt_align; /* use worst case alignment */2891 char tt_pad[KMP_PAD(kmp_base_task_team_t, CACHE_LINE)];2892};2893 2894typedef struct kmp_task_team_list_t {2895 kmp_task_team_t *task_team;2896 kmp_task_team_list_t *next;2897} kmp_task_team_list_t;2898 2899#if (USE_FAST_MEMORY == 3) || (USE_FAST_MEMORY == 5)2900// Free lists keep same-size free memory slots for fast memory allocation2901// routines2902typedef struct kmp_free_list {2903 void *th_free_list_self; // Self-allocated tasks free list2904 void *th_free_list_sync; // Self-allocated tasks stolen/returned by other2905 // threads2906 void *th_free_list_other; // Non-self free list (to be returned to owner's2907 // sync list)2908} kmp_free_list_t;2909#endif2910// Hot teams array keeps hot teams and their sizes for given thread. Hot teams2911// are not put in teams pool, and they don't put threads in threads pool.2912typedef struct kmp_hot_team_ptr {2913 kmp_team_p *hot_team; // pointer to hot_team of given nesting level2914 kmp_int32 hot_team_nth; // number of threads allocated for the hot_team2915} kmp_hot_team_ptr_t;2916typedef struct kmp_teams_size {2917 kmp_int32 nteams; // number of teams in a league2918 kmp_int32 nth; // number of threads in each team of the league2919} kmp_teams_size_t;2920 2921// This struct stores a thread that acts as a "root" for a contention2922// group. Contention groups are rooted at kmp_root threads, but also at2923// each primary thread of each team created in the teams construct.2924// This struct therefore also stores a thread_limit associated with2925// that contention group, and a counter to track the number of threads2926// active in that contention group. Each thread has a list of these: CG2927// root threads have an entry in their list in which cg_root refers to2928// the thread itself, whereas other workers in the CG will have a2929// single entry where cg_root is same as the entry containing their CG2930// root. When a thread encounters a teams construct, it will add a new2931// entry to the front of its list, because it now roots a new CG.2932typedef struct kmp_cg_root {2933 kmp_info_p *cg_root; // "root" thread for a contention group2934 // The CG root's limit comes from OMP_THREAD_LIMIT for root threads, or2935 // thread_limit clause for teams primary threads2936 kmp_int32 cg_thread_limit;2937 kmp_int32 cg_nthreads; // Count of active threads in CG rooted at cg_root2938 struct kmp_cg_root *up; // pointer to higher level CG root in list2939} kmp_cg_root_t;2940 2941// OpenMP thread data structures2942 2943typedef struct KMP_ALIGN_CACHE kmp_base_info {2944 /* Start with the readonly data which is cache aligned and padded. This is2945 written before the thread starts working by the primary thread. Uber2946 masters may update themselves later. Usage does not consider serialized2947 regions. */2948 kmp_desc_t th_info;2949 kmp_team_p *th_team; /* team we belong to */2950 kmp_root_p *th_root; /* pointer to root of task hierarchy */2951 kmp_info_p *th_next_pool; /* next available thread in the pool */2952 kmp_disp_t *th_dispatch; /* thread's dispatch data */2953 int th_in_pool; /* in thread pool (32 bits for TCR/TCW) */2954 2955 /* The following are cached from the team info structure */2956 /* TODO use these in more places as determined to be needed via profiling */2957 int th_team_nproc; /* number of threads in a team */2958 kmp_info_p *th_team_master; /* the team's primary thread */2959 int th_team_serialized; /* team is serialized */2960 microtask_t th_teams_microtask; /* save entry address for teams construct */2961 int th_teams_level; /* save initial level of teams construct */2962/* it is 0 on device but may be any on host */2963 2964/* The blocktime info is copied from the team struct to the thread struct */2965/* at the start of a barrier, and the values stored in the team are used */2966/* at points in the code where the team struct is no longer guaranteed */2967/* to exist (from the POV of worker threads). */2968#if KMP_USE_MONITOR2969 int th_team_bt_intervals;2970 int th_team_bt_set;2971#else2972 kmp_uint64 th_team_bt_intervals;2973#endif2974 2975#if KMP_AFFINITY_SUPPORTED2976 kmp_affin_mask_t *th_affin_mask; /* thread's current affinity mask */2977 kmp_affinity_ids_t th_topology_ids; /* thread's current topology ids */2978 kmp_affinity_attrs_t th_topology_attrs; /* thread's current topology attrs */2979#endif2980 omp_allocator_handle_t th_def_allocator; /* default allocator */2981 /* The data set by the primary thread at reinit, then R/W by the worker */2982 KMP_ALIGN_CACHE int2983 th_set_nproc; /* if > 0, then only use this request for the next fork */2984 int *th_set_nested_nth;2985 bool th_nt_strict; // num_threads clause has strict modifier2986 ident_t *th_nt_loc; // loc for strict modifier2987 int th_nt_sev; // error severity for strict modifier2988 const char *th_nt_msg; // error message for strict modifier2989 int th_set_nested_nth_sz;2990 kmp_hot_team_ptr_t *th_hot_teams; /* array of hot teams */2991 kmp_proc_bind_t2992 th_set_proc_bind; /* if != proc_bind_default, use request for next fork */2993 kmp_teams_size_t2994 th_teams_size; /* number of teams/threads in teams construct */2995#if KMP_AFFINITY_SUPPORTED2996 int th_current_place; /* place currently bound to */2997 int th_new_place; /* place to bind to in par reg */2998 int th_first_place; /* first place in partition */2999 int th_last_place; /* last place in partition */3000#endif3001 int th_prev_level; /* previous level for affinity format */3002 int th_prev_num_threads; /* previous num_threads for affinity format */3003#if USE_ITT_BUILD3004 kmp_uint64 th_bar_arrive_time; /* arrival to barrier timestamp */3005 kmp_uint64 th_bar_min_time; /* minimum arrival time at the barrier */3006 kmp_uint64 th_frame_time; /* frame timestamp */3007#endif /* USE_ITT_BUILD */3008 kmp_local_t th_local;3009 struct private_common *th_pri_head;3010 3011 /* Now the data only used by the worker (after initial allocation) */3012 /* TODO the first serial team should actually be stored in the info_t3013 structure. this will help reduce initial allocation overhead */3014 KMP_ALIGN_CACHE kmp_team_p3015 *th_serial_team; /*serialized team held in reserve*/3016 3017#if OMPT_SUPPORT3018 ompt_thread_info_t ompt_thread_info;3019#endif3020 3021 /* The following are also read by the primary thread during reinit */3022 struct common_table *th_pri_common;3023 3024 volatile kmp_uint32 th_spin_here; /* thread-local location for spinning */3025 /* while awaiting queuing lock acquire */3026 3027 volatile void *th_sleep_loc; // this points at a kmp_flag<T>3028 flag_type th_sleep_loc_type; // enum type of flag stored in th_sleep_loc3029 3030 ident_t *th_ident;3031 unsigned th_x; // Random number generator data3032 unsigned th_a; // Random number generator data3033 3034 /* Tasking-related data for the thread */3035 kmp_task_team_t *th_task_team; // Task team struct3036 kmp_taskdata_t *th_current_task; // Innermost Task being executed3037 kmp_uint8 th_task_state; // alternating 0/1 for task team identification3038 kmp_uint32 th_reap_state; // Non-zero indicates thread is not3039 // tasking, thus safe to reap3040 3041 /* More stuff for keeping track of active/sleeping threads (this part is3042 written by the worker thread) */3043 kmp_uint8 th_active_in_pool; // included in count of #active threads in pool3044 int th_active; // ! sleeping; 32 bits for TCR/TCW3045 std::atomic<kmp_uint32> th_used_in_team; // Flag indicating use in team3046 // 0 = not used in team; 1 = used in team;3047 // 2 = transitioning to not used in team; 3 = transitioning to used in team3048 struct cons_header *th_cons; // used for consistency check3049#if KMP_USE_HIER_SCHED3050 // used for hierarchical scheduling3051 kmp_hier_private_bdata_t *th_hier_bar_data;3052#endif3053 3054 /* Add the syncronizing data which is cache aligned and padded. */3055 KMP_ALIGN_CACHE kmp_balign_t th_bar[bs_last_barrier];3056 3057 KMP_ALIGN_CACHE volatile kmp_int323058 th_next_waiting; /* gtid+1 of next thread on lock wait queue, 0 if none */3059 3060#if (USE_FAST_MEMORY == 3) || (USE_FAST_MEMORY == 5)3061#define NUM_LISTS 43062 kmp_free_list_t th_free_lists[NUM_LISTS]; // Free lists for fast memory3063// allocation routines3064#endif3065 3066#if KMP_OS_WINDOWS3067 kmp_win32_cond_t th_suspend_cv;3068 kmp_win32_mutex_t th_suspend_mx;3069 std::atomic<int> th_suspend_init;3070#endif3071#if KMP_OS_UNIX3072 kmp_cond_align_t th_suspend_cv;3073 kmp_mutex_align_t th_suspend_mx;3074 std::atomic<int> th_suspend_init_count;3075#endif3076 3077#if USE_ITT_BUILD3078 kmp_itt_mark_t th_itt_mark_single;3079// alignment ???3080#endif /* USE_ITT_BUILD */3081#if KMP_STATS_ENABLED3082 kmp_stats_list *th_stats;3083#endif3084#if KMP_OS_UNIX3085 std::atomic<bool> th_blocking;3086#endif3087 kmp_cg_root_t *th_cg_roots; // list of cg_roots associated with this thread3088} kmp_base_info_t;3089 3090typedef union KMP_ALIGN_CACHE kmp_info {3091 double th_align; /* use worst case alignment */3092 char th_pad[KMP_PAD(kmp_base_info_t, CACHE_LINE)];3093 kmp_base_info_t th;3094} kmp_info_t;3095 3096// OpenMP thread team data structures3097 3098typedef struct kmp_base_data {3099 volatile kmp_uint32 t_value;3100} kmp_base_data_t;3101 3102typedef union KMP_ALIGN_CACHE kmp_sleep_team {3103 double dt_align; /* use worst case alignment */3104 char dt_pad[KMP_PAD(kmp_base_data_t, CACHE_LINE)];3105 kmp_base_data_t dt;3106} kmp_sleep_team_t;3107 3108typedef union KMP_ALIGN_CACHE kmp_ordered_team {3109 double dt_align; /* use worst case alignment */3110 char dt_pad[KMP_PAD(kmp_base_data_t, CACHE_LINE)];3111 kmp_base_data_t dt;3112} kmp_ordered_team_t;3113 3114typedef int (*launch_t)(int gtid);3115 3116/* Minimum number of ARGV entries to malloc if necessary */3117#define KMP_MIN_MALLOC_ARGV_ENTRIES 1003118 3119// Set up how many argv pointers will fit in cache lines containing3120// t_inline_argv. Historically, we have supported at least 96 bytes. Using a3121// larger value for more space between the primary write/worker read section and3122// read/write by all section seems to buy more performance on EPCC PARALLEL.3123#if KMP_ARCH_X86 || KMP_ARCH_X86_643124#define KMP_INLINE_ARGV_BYTES \3125 (4 * CACHE_LINE - \3126 ((3 * KMP_PTR_SKIP + 2 * sizeof(int) + 2 * sizeof(kmp_int8) + \3127 sizeof(kmp_int16) + sizeof(kmp_uint32)) % \3128 CACHE_LINE))3129#else3130#define KMP_INLINE_ARGV_BYTES \3131 (2 * CACHE_LINE - ((3 * KMP_PTR_SKIP + 2 * sizeof(int)) % CACHE_LINE))3132#endif3133#define KMP_INLINE_ARGV_ENTRIES (int)(KMP_INLINE_ARGV_BYTES / KMP_PTR_SKIP)3134 3135typedef struct KMP_ALIGN_CACHE kmp_base_team {3136 // Synchronization Data3137 // ---------------------------------------------------------------------------3138 KMP_ALIGN_CACHE kmp_ordered_team_t t_ordered;3139 kmp_balign_team_t t_bar[bs_last_barrier];3140 std::atomic<int> t_construct; // count of single directive encountered by team3141 char pad[sizeof(kmp_lock_t)]; // padding to maintain performance on big iron3142 3143 // [0] - parallel / [1] - worksharing task reduction data shared by taskgroups3144 std::atomic<void *> t_tg_reduce_data[2]; // to support task modifier3145 std::atomic<int> t_tg_fini_counter[2]; // sync end of task reductions3146 3147 // Primary thread only3148 // ---------------------------------------------------------------------------3149 KMP_ALIGN_CACHE int t_master_tid; // tid of primary thread in parent team3150 int t_master_this_cons; // "this_construct" single counter of primary thread3151 // in parent team3152 ident_t *t_ident; // if volatile, have to change too much other crud to3153 // volatile too3154 kmp_team_p *t_parent; // parent team3155 kmp_team_p *t_next_pool; // next free team in the team pool3156 kmp_disp_t *t_dispatch; // thread's dispatch data3157 kmp_task_team_t *t_task_team[2]; // Task team struct; switch between 23158 kmp_proc_bind_t t_proc_bind; // bind type for par region3159 int t_primary_task_state; // primary thread's task state saved3160#if USE_ITT_BUILD3161 kmp_uint64 t_region_time; // region begin timestamp3162#endif /* USE_ITT_BUILD */3163 3164 // Primary thread write, workers read3165 // --------------------------------------------------------------------------3166 KMP_ALIGN_CACHE void **t_argv;3167 int t_argc;3168 int t_nproc; // number of threads in team3169 microtask_t t_pkfn;3170 launch_t t_invoke; // procedure to launch the microtask3171 3172#if OMPT_SUPPORT3173 ompt_team_info_t ompt_team_info;3174 ompt_lw_taskteam_t *ompt_serialized_team_info;3175#endif3176 3177#if KMP_ARCH_X86 || KMP_ARCH_X86_643178 kmp_int8 t_fp_control_saved;3179 kmp_int8 t_pad2b;3180 kmp_int16 t_x87_fpu_control_word; // FP control regs3181 kmp_uint32 t_mxcsr;3182#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */3183 3184 void *t_inline_argv[KMP_INLINE_ARGV_ENTRIES];3185 3186 KMP_ALIGN_CACHE kmp_info_t **t_threads;3187 kmp_taskdata_t3188 *t_implicit_task_taskdata; // Taskdata for the thread's implicit task3189 int t_level; // nested parallel level3190 3191 KMP_ALIGN_CACHE int t_max_argc;3192 int t_max_nproc; // max threads this team can handle (dynamically expandable)3193 int t_serialized; // levels deep of serialized teams3194 dispatch_shared_info_t *t_disp_buffer; // buffers for dispatch system3195 int t_id; // team's id, assigned by debugger.3196 int t_active_level; // nested active parallel level3197 kmp_r_sched_t t_sched; // run-time schedule for the team3198#if KMP_AFFINITY_SUPPORTED3199 int t_first_place; // first & last place in parent thread's partition.3200 int t_last_place; // Restore these values to primary thread after par region.3201#endif // KMP_AFFINITY_SUPPORTED3202 int t_display_affinity;3203 int t_size_changed; // team size was changed?: 0: no, 1: yes, -1: changed via3204 // omp_set_num_threads() call3205 omp_allocator_handle_t t_def_allocator; /* default allocator */3206 3207// Read/write by workers as well3208#if (KMP_ARCH_X86 || KMP_ARCH_X86_64)3209 // Using CACHE_LINE=64 reduces memory footprint, but causes a big perf3210 // regression of epcc 'parallel' and 'barrier' on fxe256lin01. This extra3211 // padding serves to fix the performance of epcc 'parallel' and 'barrier' when3212 // CACHE_LINE=64. TODO: investigate more and get rid if this padding.3213 char dummy_padding[1024];3214#endif3215 // Internal control stack for additional nested teams.3216 KMP_ALIGN_CACHE kmp_internal_control_t *t_control_stack_top;3217 // for SERIALIZED teams nested 2 or more levels deep3218 // typed flag to store request state of cancellation3219 std::atomic<kmp_int32> t_cancel_request;3220 int t_master_active; // save on fork, restore on join3221 void *t_copypriv_data; // team specific pointer to copyprivate data array3222#if KMP_OS_WINDOWS3223 std::atomic<kmp_uint32> t_copyin_counter;3224#endif3225#if USE_ITT_BUILD3226 void *t_stack_id; // team specific stack stitching id (for ittnotify)3227#endif /* USE_ITT_BUILD */3228 distributedBarrier *b; // Distributed barrier data associated with team3229 kmp_nested_nthreads_t *t_nested_nth;3230} kmp_base_team_t;3231 3232// Assert that the list structure fits and aligns within3233// the double task team pointer3234KMP_BUILD_ASSERT(sizeof(kmp_task_team_t *[2]) == sizeof(kmp_task_team_list_t));3235KMP_BUILD_ASSERT(alignof(kmp_task_team_t *[2]) ==3236 alignof(kmp_task_team_list_t));3237 3238union KMP_ALIGN_CACHE kmp_team {3239 kmp_base_team_t t;3240 double t_align; /* use worst case alignment */3241 char t_pad[KMP_PAD(kmp_base_team_t, CACHE_LINE)];3242};3243 3244typedef union KMP_ALIGN_CACHE kmp_time_global {3245 double dt_align; /* use worst case alignment */3246 char dt_pad[KMP_PAD(kmp_base_data_t, CACHE_LINE)];3247 kmp_base_data_t dt;3248} kmp_time_global_t;3249 3250typedef struct kmp_base_global {3251 /* cache-aligned */3252 kmp_time_global_t g_time;3253 3254 /* non cache-aligned */3255 volatile int g_abort;3256 volatile int g_done;3257 3258 int g_dynamic;3259 enum dynamic_mode g_dynamic_mode;3260} kmp_base_global_t;3261 3262typedef union KMP_ALIGN_CACHE kmp_global {3263 kmp_base_global_t g;3264 double g_align; /* use worst case alignment */3265 char g_pad[KMP_PAD(kmp_base_global_t, CACHE_LINE)];3266} kmp_global_t;3267 3268typedef struct kmp_base_root {3269 // TODO: GEH - combine r_active with r_in_parallel then r_active ==3270 // (r_in_parallel>= 0)3271 // TODO: GEH - then replace r_active with t_active_levels if we can to reduce3272 // the synch overhead or keeping r_active3273 volatile int r_active; /* TRUE if some region in a nest has > 1 thread */3274 // keeps a count of active parallel regions per root3275 std::atomic<int> r_in_parallel;3276 // GEH: This is misnamed, should be r_active_levels3277 kmp_team_t *r_root_team;3278 kmp_team_t *r_hot_team;3279 kmp_info_t *r_uber_thread;3280 kmp_lock_t r_begin_lock;3281 volatile int r_begin;3282 int r_blocktime; /* blocktime for this root and descendants */3283#if KMP_AFFINITY_SUPPORTED3284 int r_affinity_assigned;3285#endif // KMP_AFFINITY_SUPPORTED3286} kmp_base_root_t;3287 3288typedef union KMP_ALIGN_CACHE kmp_root {3289 kmp_base_root_t r;3290 double r_align; /* use worst case alignment */3291 char r_pad[KMP_PAD(kmp_base_root_t, CACHE_LINE)];3292} kmp_root_t;3293 3294struct fortran_inx_info {3295 kmp_int32 data;3296};3297 3298// This list type exists to hold old __kmp_threads arrays so that3299// old references to them may complete while reallocation takes place when3300// expanding the array. The items in this list are kept alive until library3301// shutdown.3302typedef struct kmp_old_threads_list_t {3303 kmp_info_t **threads;3304 struct kmp_old_threads_list_t *next;3305} kmp_old_threads_list_t;3306 3307/* ------------------------------------------------------------------------ */3308 3309extern int __kmp_settings;3310extern int __kmp_duplicate_library_ok;3311#if USE_ITT_BUILD3312extern int __kmp_forkjoin_frames;3313extern int __kmp_forkjoin_frames_mode;3314#endif3315extern PACKED_REDUCTION_METHOD_T __kmp_force_reduction_method;3316extern int __kmp_determ_red;3317 3318#ifdef KMP_DEBUG3319extern int kmp_a_debug;3320extern int kmp_b_debug;3321extern int kmp_c_debug;3322extern int kmp_d_debug;3323extern int kmp_e_debug;3324extern int kmp_f_debug;3325#endif /* KMP_DEBUG */3326 3327/* For debug information logging using rotating buffer */3328#define KMP_DEBUG_BUF_LINES_INIT 5123329#define KMP_DEBUG_BUF_LINES_MIN 13330 3331#define KMP_DEBUG_BUF_CHARS_INIT 1283332#define KMP_DEBUG_BUF_CHARS_MIN 23333 3334extern int3335 __kmp_debug_buf; /* TRUE means use buffer, FALSE means print to stderr */3336extern int __kmp_debug_buf_lines; /* How many lines of debug stored in buffer */3337extern int3338 __kmp_debug_buf_chars; /* How many characters allowed per line in buffer */3339extern int __kmp_debug_buf_atomic; /* TRUE means use atomic update of buffer3340 entry pointer */3341 3342extern char *__kmp_debug_buffer; /* Debug buffer itself */3343extern std::atomic<int> __kmp_debug_count; /* Counter for number of lines3344 printed in buffer so far */3345extern int __kmp_debug_buf_warn_chars; /* Keep track of char increase3346 recommended in warnings */3347/* end rotating debug buffer */3348 3349#ifdef KMP_DEBUG3350extern int __kmp_par_range; /* +1 => only go par for constructs in range */3351 3352#define KMP_PAR_RANGE_ROUTINE_LEN 10243353extern char __kmp_par_range_routine[KMP_PAR_RANGE_ROUTINE_LEN];3354#define KMP_PAR_RANGE_FILENAME_LEN 10243355extern char __kmp_par_range_filename[KMP_PAR_RANGE_FILENAME_LEN];3356extern int __kmp_par_range_lb;3357extern int __kmp_par_range_ub;3358#endif3359 3360/* For printing out dynamic storage map for threads and teams */3361extern int3362 __kmp_storage_map; /* True means print storage map for threads and teams */3363extern int __kmp_storage_map_verbose; /* True means storage map includes3364 placement info */3365extern int __kmp_storage_map_verbose_specified;3366 3367#if KMP_ARCH_X86 || KMP_ARCH_X86_643368extern kmp_cpuinfo_t __kmp_cpuinfo;3369static inline bool __kmp_is_hybrid_cpu() { return __kmp_cpuinfo.flags.hybrid; }3370#elif KMP_OS_DARWIN && KMP_ARCH_AARCH643371static inline bool __kmp_is_hybrid_cpu() { return true; }3372#else3373static inline bool __kmp_is_hybrid_cpu() { return false; }3374#endif3375 3376extern volatile int __kmp_init_serial;3377extern volatile int __kmp_init_gtid;3378extern volatile int __kmp_init_common;3379extern volatile int __kmp_need_register_serial;3380extern volatile int __kmp_init_middle;3381extern volatile int __kmp_init_parallel;3382#if KMP_USE_MONITOR3383extern volatile int __kmp_init_monitor;3384#endif3385extern volatile int __kmp_init_user_locks;3386extern volatile int __kmp_init_hidden_helper_threads;3387extern int __kmp_init_counter;3388extern int __kmp_root_counter;3389extern int __kmp_version;3390 3391/* list of address of allocated caches for commons */3392extern kmp_cached_addr_t *__kmp_threadpriv_cache_list;3393 3394/* Barrier algorithm types and options */3395extern kmp_uint32 __kmp_barrier_gather_bb_dflt;3396extern kmp_uint32 __kmp_barrier_release_bb_dflt;3397extern kmp_bar_pat_e __kmp_barrier_gather_pat_dflt;3398extern kmp_bar_pat_e __kmp_barrier_release_pat_dflt;3399extern kmp_uint32 __kmp_barrier_gather_branch_bits[bs_last_barrier];3400extern kmp_uint32 __kmp_barrier_release_branch_bits[bs_last_barrier];3401extern kmp_bar_pat_e __kmp_barrier_gather_pattern[bs_last_barrier];3402extern kmp_bar_pat_e __kmp_barrier_release_pattern[bs_last_barrier];3403extern char const *__kmp_barrier_branch_bit_env_name[bs_last_barrier];3404extern char const *__kmp_barrier_pattern_env_name[bs_last_barrier];3405extern char const *__kmp_barrier_type_name[bs_last_barrier];3406extern char const *__kmp_barrier_pattern_name[bp_last_bar];3407 3408/* Global Locks */3409extern kmp_bootstrap_lock_t __kmp_initz_lock; /* control initialization */3410extern kmp_bootstrap_lock_t __kmp_forkjoin_lock; /* control fork/join access */3411extern kmp_bootstrap_lock_t __kmp_task_team_lock;3412extern kmp_bootstrap_lock_t3413 __kmp_exit_lock; /* exit() is not always thread-safe */3414#if KMP_USE_MONITOR3415extern kmp_bootstrap_lock_t3416 __kmp_monitor_lock; /* control monitor thread creation */3417#endif3418extern kmp_bootstrap_lock_t3419 __kmp_tp_cached_lock; /* used for the hack to allow threadprivate cache and3420 __kmp_threads expansion to co-exist */3421 3422extern kmp_lock_t __kmp_global_lock; /* control OS/global access */3423 3424extern enum library_type __kmp_library;3425 3426extern enum sched_type __kmp_sched; /* default runtime scheduling */3427extern enum sched_type __kmp_static; /* default static scheduling method */3428extern enum sched_type __kmp_guided; /* default guided scheduling method */3429extern enum sched_type __kmp_auto; /* default auto scheduling method */3430extern int __kmp_chunk; /* default runtime chunk size */3431extern int __kmp_force_monotonic; /* whether monotonic scheduling forced */3432 3433extern size_t __kmp_stksize; /* stack size per thread */3434#if KMP_USE_MONITOR3435extern size_t __kmp_monitor_stksize; /* stack size for monitor thread */3436#endif3437extern size_t __kmp_stkoffset; /* stack offset per thread */3438extern int __kmp_stkpadding; /* Should we pad root thread(s) stack */3439 3440extern size_t3441 __kmp_malloc_pool_incr; /* incremental size of pool for kmp_malloc() */3442extern int __kmp_env_stksize; /* was KMP_STACKSIZE specified? */3443extern int __kmp_env_blocktime; /* was KMP_BLOCKTIME specified? */3444extern int __kmp_env_checks; /* was KMP_CHECKS specified? */3445extern int __kmp_env_consistency_check; // was KMP_CONSISTENCY_CHECK specified?3446extern int __kmp_generate_warnings; /* should we issue warnings? */3447extern int __kmp_reserve_warn; /* have we issued reserve_threads warning? */3448 3449#ifdef DEBUG_SUSPEND3450extern int __kmp_suspend_count; /* count inside __kmp_suspend_template() */3451#endif3452 3453extern kmp_int32 __kmp_use_yield;3454extern kmp_int32 __kmp_use_yield_exp_set;3455extern kmp_uint32 __kmp_yield_init;3456extern kmp_uint32 __kmp_yield_next;3457extern kmp_uint64 __kmp_pause_init;3458 3459/* ------------------------------------------------------------------------- */3460extern int __kmp_allThreadsSpecified;3461 3462extern size_t __kmp_align_alloc;3463/* following data protected by initialization routines */3464extern int __kmp_xproc; /* number of processors in the system */3465extern int __kmp_avail_proc; /* number of processors available to the process */3466extern size_t __kmp_sys_min_stksize; /* system-defined minimum stack size */3467extern int __kmp_sys_max_nth; /* system-imposed maximum number of threads */3468// maximum total number of concurrently-existing threads on device3469extern int __kmp_max_nth;3470// maximum total number of concurrently-existing threads in a contention group3471extern int __kmp_cg_max_nth;3472extern int __kmp_task_max_nth; // max threads used in a task3473extern int __kmp_teams_max_nth; // max threads used in a teams construct3474extern int __kmp_threads_capacity; /* capacity of the arrays __kmp_threads and3475 __kmp_root */3476extern int __kmp_dflt_team_nth; /* default number of threads in a parallel3477 region a la OMP_NUM_THREADS */3478extern int __kmp_dflt_team_nth_ub; /* upper bound on "" determined at serial3479 initialization */3480extern int __kmp_tp_capacity; /* capacity of __kmp_threads if threadprivate is3481 used (fixed) */3482extern int __kmp_tp_cached; /* whether threadprivate cache has been created3483 (__kmpc_threadprivate_cached()) */3484extern int __kmp_dflt_blocktime; /* number of microseconds to wait before3485 blocking (env setting) */3486extern char __kmp_blocktime_units; /* 'm' or 'u' to note units specified */3487extern bool __kmp_wpolicy_passive; /* explicitly set passive wait policy */3488 3489// Convert raw blocktime from ms to us if needed.3490static inline void __kmp_aux_convert_blocktime(int *bt) {3491 if (__kmp_blocktime_units == 'm') {3492 if (*bt > INT_MAX / 1000) {3493 *bt = INT_MAX / 1000;3494 KMP_INFORM(MaxValueUsing, "kmp_set_blocktime(ms)", bt);3495 }3496 *bt = *bt * 1000;3497 }3498}3499 3500#if KMP_USE_MONITOR3501extern int3502 __kmp_monitor_wakeups; /* number of times monitor wakes up per second */3503extern int __kmp_bt_intervals; /* number of monitor timestamp intervals before3504 blocking */3505#endif3506#ifdef KMP_ADJUST_BLOCKTIME3507extern int __kmp_zero_bt; /* whether blocktime has been forced to zero */3508#endif /* KMP_ADJUST_BLOCKTIME */3509#ifdef KMP_DFLT_NTH_CORES3510extern int __kmp_ncores; /* Total number of cores for threads placement */3511#endif3512/* Number of millisecs to delay on abort for Intel(R) VTune(TM) tools */3513extern int __kmp_abort_delay;3514 3515extern int __kmp_need_register_atfork_specified;3516extern int __kmp_need_register_atfork; /* At initialization, call pthread_atfork3517 to install fork handler */3518extern int __kmp_gtid_mode; /* Method of getting gtid, values:3519 0 - not set, will be set at runtime3520 1 - using stack search3521 2 - dynamic TLS (pthread_getspecific(Linux* OS/OS3522 X*) or TlsGetValue(Windows* OS))3523 3 - static TLS (__declspec(thread) __kmp_gtid),3524 Linux* OS .so only. */3525extern int3526 __kmp_adjust_gtid_mode; /* If true, adjust method based on #threads */3527#ifdef KMP_TDATA_GTID3528extern KMP_THREAD_LOCAL int __kmp_gtid;3529#endif3530extern int __kmp_tls_gtid_min; /* #threads below which use sp search for gtid */3531extern int __kmp_foreign_tp; // If true, separate TP var for each foreign thread3532#if KMP_ARCH_X86 || KMP_ARCH_X86_643533extern int __kmp_inherit_fp_control; // copy fp creg(s) parent->workers at fork3534extern kmp_int16 __kmp_init_x87_fpu_control_word; // init thread's FP ctrl reg3535extern kmp_uint32 __kmp_init_mxcsr; /* init thread's mxscr */3536#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */3537 3538// max_active_levels for nested parallelism enabled by default via3539// OMP_MAX_ACTIVE_LEVELS, OMP_NESTED, OMP_NUM_THREADS, and OMP_PROC_BIND3540extern int __kmp_dflt_max_active_levels;3541// Indicates whether value of __kmp_dflt_max_active_levels was already3542// explicitly set by OMP_MAX_ACTIVE_LEVELS or OMP_NESTED=false3543extern bool __kmp_dflt_max_active_levels_set;3544extern int __kmp_dispatch_num_buffers; /* max possible dynamic loops in3545 concurrent execution per team */3546extern int __kmp_hot_teams_mode;3547extern int __kmp_hot_teams_max_level;3548 3549#if KMP_MIC_SUPPORTED3550extern enum mic_type __kmp_mic_type;3551#endif3552 3553#ifdef USE_LOAD_BALANCE3554extern double __kmp_load_balance_interval; // load balance algorithm interval3555#endif /* USE_LOAD_BALANCE */3556 3557#if KMP_USE_ADAPTIVE_LOCKS3558 3559// Parameters for the speculative lock backoff system.3560struct kmp_adaptive_backoff_params_t {3561 // Number of soft retries before it counts as a hard retry.3562 kmp_uint32 max_soft_retries;3563 // Badness is a bit mask : 0,1,3,7,15,... on each hard failure we move one to3564 // the right3565 kmp_uint32 max_badness;3566};3567 3568extern kmp_adaptive_backoff_params_t __kmp_adaptive_backoff_params;3569 3570#if KMP_DEBUG_ADAPTIVE_LOCKS3571extern const char *__kmp_speculative_statsfile;3572#endif3573 3574#endif // KMP_USE_ADAPTIVE_LOCKS3575 3576extern int __kmp_display_env; /* TRUE or FALSE */3577extern int __kmp_display_env_verbose; /* TRUE if OMP_DISPLAY_ENV=VERBOSE */3578extern int __kmp_omp_cancellation; /* TRUE or FALSE */3579extern int __kmp_nteams;3580extern int __kmp_teams_thread_limit;3581 3582/* ------------------------------------------------------------------------- */3583 3584/* the following are protected by the fork/join lock */3585/* write: lock read: anytime */3586extern kmp_info_t **__kmp_threads; /* Descriptors for the threads */3587/* Holds old arrays of __kmp_threads until library shutdown */3588extern kmp_old_threads_list_t *__kmp_old_threads_list;3589/* read/write: lock */3590extern volatile kmp_team_t *__kmp_team_pool;3591extern volatile kmp_info_t *__kmp_thread_pool;3592extern kmp_info_t *__kmp_thread_pool_insert_pt;3593 3594// total num threads reachable from some root thread including all root threads3595extern volatile int __kmp_nth;3596/* total number of threads reachable from some root thread including all root3597 threads, and those in the thread pool */3598extern volatile int __kmp_all_nth;3599extern std::atomic<int> __kmp_thread_pool_active_nth;3600 3601extern kmp_root_t **__kmp_root; /* root of thread hierarchy */3602/* end data protected by fork/join lock */3603/* ------------------------------------------------------------------------- */3604 3605#define __kmp_get_gtid() __kmp_get_global_thread_id()3606#define __kmp_entry_gtid() __kmp_get_global_thread_id_reg()3607#define __kmp_get_tid() (__kmp_tid_from_gtid(__kmp_get_gtid()))3608#define __kmp_get_team() (__kmp_threads[(__kmp_get_gtid())]->th.th_team)3609#define __kmp_get_thread() (__kmp_thread_from_gtid(__kmp_get_gtid()))3610 3611// AT: Which way is correct?3612// AT: 1. nproc = __kmp_threads[ ( gtid ) ] -> th.th_team -> t.t_nproc;3613// AT: 2. nproc = __kmp_threads[ ( gtid ) ] -> th.th_team_nproc;3614#define __kmp_get_team_num_threads(gtid) \3615 (__kmp_threads[(gtid)]->th.th_team->t.t_nproc)3616 3617static inline bool KMP_UBER_GTID(int gtid) {3618 KMP_DEBUG_ASSERT(gtid >= KMP_GTID_MIN);3619 KMP_DEBUG_ASSERT(gtid < __kmp_threads_capacity);3620 return (gtid >= 0 && __kmp_root[gtid] && __kmp_threads[gtid] &&3621 __kmp_threads[gtid] == __kmp_root[gtid]->r.r_uber_thread);3622}3623 3624static inline int __kmp_tid_from_gtid(int gtid) {3625 KMP_DEBUG_ASSERT(gtid >= 0);3626 return __kmp_threads[gtid]->th.th_info.ds.ds_tid;3627}3628 3629static inline int __kmp_gtid_from_tid(int tid, const kmp_team_t *team) {3630 KMP_DEBUG_ASSERT(tid >= 0 && team);3631 return team->t.t_threads[tid]->th.th_info.ds.ds_gtid;3632}3633 3634static inline int __kmp_gtid_from_thread(const kmp_info_t *thr) {3635 KMP_DEBUG_ASSERT(thr);3636 return thr->th.th_info.ds.ds_gtid;3637}3638 3639static inline kmp_info_t *__kmp_thread_from_gtid(int gtid) {3640 KMP_DEBUG_ASSERT(gtid >= 0);3641 return __kmp_threads[gtid];3642}3643 3644static inline kmp_team_t *__kmp_team_from_gtid(int gtid) {3645 KMP_DEBUG_ASSERT(gtid >= 0);3646 return __kmp_threads[gtid]->th.th_team;3647}3648 3649static inline void __kmp_assert_valid_gtid(kmp_int32 gtid) {3650 if (UNLIKELY(gtid < 0 || gtid >= __kmp_threads_capacity))3651 KMP_FATAL(ThreadIdentInvalid);3652}3653 3654#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT3655extern int __kmp_user_level_mwait; // TRUE or FALSE; from KMP_USER_LEVEL_MWAIT3656extern int __kmp_umwait_enabled; // Runtime check if user-level mwait enabled3657extern int __kmp_mwait_enabled; // Runtime check if ring3 mwait is enabled3658extern int __kmp_mwait_hints; // Hints to pass in to mwait3659#endif3660 3661#if KMP_HAVE_UMWAIT3662extern int __kmp_waitpkg_enabled; // Runtime check if waitpkg exists3663extern int __kmp_tpause_state; // 0 (default), 1=C0.1, 2=C0.2; from KMP_TPAUSE3664extern int __kmp_tpause_hint; // 1=C0.1 (default), 0=C0.2; from KMP_TPAUSE3665extern int __kmp_tpause_enabled; // 0 (default), 1 (KMP_TPAUSE is non-zero)3666#endif3667 3668/* ------------------------------------------------------------------------- */3669 3670extern kmp_global_t __kmp_global; /* global status */3671 3672extern kmp_info_t __kmp_monitor;3673// For Debugging Support Library3674extern std::atomic<kmp_int32> __kmp_team_counter;3675// For Debugging Support Library3676extern std::atomic<kmp_int32> __kmp_task_counter;3677 3678#if USE_DEBUGGER3679#define _KMP_GEN_ID(counter) \3680 (__kmp_debugging ? KMP_ATOMIC_INC(&counter) + 1 : ~0)3681#else3682#define _KMP_GEN_ID(counter) (~0)3683#endif /* USE_DEBUGGER */3684 3685#define KMP_GEN_TASK_ID() _KMP_GEN_ID(__kmp_task_counter)3686#define KMP_GEN_TEAM_ID() _KMP_GEN_ID(__kmp_team_counter)3687 3688/* ------------------------------------------------------------------------ */3689 3690extern void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2,3691 size_t size, char const *format, ...);3692 3693extern void __kmp_serial_initialize(void);3694extern void __kmp_middle_initialize(void);3695extern void __kmp_parallel_initialize(void);3696 3697extern void __kmp_internal_begin(void);3698extern void __kmp_internal_end_library(int gtid);3699extern void __kmp_internal_end_thread(int gtid);3700extern void __kmp_internal_end_atexit(void);3701extern void __kmp_internal_end_dtor(void);3702extern void __kmp_internal_end_dest(void *);3703 3704extern int __kmp_register_root(int initial_thread);3705extern void __kmp_unregister_root(int gtid);3706extern void __kmp_unregister_library(void); // called by __kmp_internal_end()3707 3708extern int __kmp_ignore_mppbeg(void);3709extern int __kmp_ignore_mppend(void);3710 3711extern int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws);3712extern void __kmp_exit_single(int gtid);3713 3714extern void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref);3715extern void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref);3716 3717#ifdef USE_LOAD_BALANCE3718extern int __kmp_get_load_balance(int);3719#endif3720 3721extern int __kmp_get_global_thread_id(void);3722extern int __kmp_get_global_thread_id_reg(void);3723extern void __kmp_exit_thread(int exit_status);3724extern void __kmp_abort(char const *format, ...);3725extern void __kmp_abort_thread(void);3726KMP_NORETURN extern void __kmp_abort_process(void);3727extern void __kmp_warn(char const *format, ...);3728 3729extern void __kmp_set_num_threads(int new_nth, int gtid);3730 3731extern bool __kmp_detect_shm();3732extern bool __kmp_detect_tmp();3733 3734// Returns current thread (pointer to kmp_info_t). Current thread *must* be3735// registered.3736static inline kmp_info_t *__kmp_entry_thread() {3737 int gtid = __kmp_entry_gtid();3738 3739 return __kmp_threads[gtid];3740}3741 3742extern void __kmp_set_max_active_levels(int gtid, int new_max_active_levels);3743extern int __kmp_get_max_active_levels(int gtid);3744extern int __kmp_get_ancestor_thread_num(int gtid, int level);3745extern int __kmp_get_team_size(int gtid, int level);3746extern void __kmp_set_schedule(int gtid, kmp_sched_t new_sched, int chunk);3747extern void __kmp_get_schedule(int gtid, kmp_sched_t *sched, int *chunk);3748 3749extern unsigned short __kmp_get_random(kmp_info_t *thread);3750extern void __kmp_init_random(kmp_info_t *thread);3751 3752extern kmp_r_sched_t __kmp_get_schedule_global(void);3753extern void __kmp_adjust_num_threads(int new_nproc);3754extern void __kmp_check_stksize(size_t *val);3755 3756extern void *___kmp_allocate(size_t size KMP_SRC_LOC_DECL);3757extern void *___kmp_page_allocate(size_t size KMP_SRC_LOC_DECL);3758extern void ___kmp_free(void *ptr KMP_SRC_LOC_DECL);3759#define __kmp_allocate(size) ___kmp_allocate((size)KMP_SRC_LOC_CURR)3760#define __kmp_page_allocate(size) ___kmp_page_allocate((size)KMP_SRC_LOC_CURR)3761#define __kmp_free(ptr) ___kmp_free((ptr)KMP_SRC_LOC_CURR)3762 3763#if USE_FAST_MEMORY3764extern void *___kmp_fast_allocate(kmp_info_t *this_thr,3765 size_t size KMP_SRC_LOC_DECL);3766extern void ___kmp_fast_free(kmp_info_t *this_thr, void *ptr KMP_SRC_LOC_DECL);3767extern void __kmp_free_fast_memory(kmp_info_t *this_thr);3768extern void __kmp_initialize_fast_memory(kmp_info_t *this_thr);3769#define __kmp_fast_allocate(this_thr, size) \3770 ___kmp_fast_allocate((this_thr), (size)KMP_SRC_LOC_CURR)3771#define __kmp_fast_free(this_thr, ptr) \3772 ___kmp_fast_free((this_thr), (ptr)KMP_SRC_LOC_CURR)3773#endif3774 3775extern void *___kmp_thread_malloc(kmp_info_t *th, size_t size KMP_SRC_LOC_DECL);3776extern void *___kmp_thread_calloc(kmp_info_t *th, size_t nelem,3777 size_t elsize KMP_SRC_LOC_DECL);3778extern void *___kmp_thread_realloc(kmp_info_t *th, void *ptr,3779 size_t size KMP_SRC_LOC_DECL);3780extern void ___kmp_thread_free(kmp_info_t *th, void *ptr KMP_SRC_LOC_DECL);3781#define __kmp_thread_malloc(th, size) \3782 ___kmp_thread_malloc((th), (size)KMP_SRC_LOC_CURR)3783#define __kmp_thread_calloc(th, nelem, elsize) \3784 ___kmp_thread_calloc((th), (nelem), (elsize)KMP_SRC_LOC_CURR)3785#define __kmp_thread_realloc(th, ptr, size) \3786 ___kmp_thread_realloc((th), (ptr), (size)KMP_SRC_LOC_CURR)3787#define __kmp_thread_free(th, ptr) \3788 ___kmp_thread_free((th), (ptr)KMP_SRC_LOC_CURR)3789 3790extern void __kmp_push_num_threads(ident_t *loc, int gtid, int num_threads);3791extern void __kmp_push_num_threads_list(ident_t *loc, int gtid,3792 kmp_uint32 list_length,3793 int *num_threads_list);3794extern void __kmp_set_strict_num_threads(ident_t *loc, int gtid, int sev,3795 const char *msg);3796 3797extern void __kmp_push_proc_bind(ident_t *loc, int gtid,3798 kmp_proc_bind_t proc_bind);3799extern void __kmp_push_num_teams(ident_t *loc, int gtid, int num_teams,3800 int num_threads);3801extern void __kmp_push_num_teams_51(ident_t *loc, int gtid, int num_teams_lb,3802 int num_teams_ub, int num_threads);3803 3804extern void __kmp_yield();3805 3806extern void __kmpc_dispatch_init_4(ident_t *loc, kmp_int32 gtid,3807 enum sched_type schedule, kmp_int32 lb,3808 kmp_int32 ub, kmp_int32 st, kmp_int32 chunk);3809extern void __kmpc_dispatch_init_4u(ident_t *loc, kmp_int32 gtid,3810 enum sched_type schedule, kmp_uint32 lb,3811 kmp_uint32 ub, kmp_int32 st,3812 kmp_int32 chunk);3813extern void __kmpc_dispatch_init_8(ident_t *loc, kmp_int32 gtid,3814 enum sched_type schedule, kmp_int64 lb,3815 kmp_int64 ub, kmp_int64 st, kmp_int64 chunk);3816extern void __kmpc_dispatch_init_8u(ident_t *loc, kmp_int32 gtid,3817 enum sched_type schedule, kmp_uint64 lb,3818 kmp_uint64 ub, kmp_int64 st,3819 kmp_int64 chunk);3820 3821extern int __kmpc_dispatch_next_4(ident_t *loc, kmp_int32 gtid,3822 kmp_int32 *p_last, kmp_int32 *p_lb,3823 kmp_int32 *p_ub, kmp_int32 *p_st);3824extern int __kmpc_dispatch_next_4u(ident_t *loc, kmp_int32 gtid,3825 kmp_int32 *p_last, kmp_uint32 *p_lb,3826 kmp_uint32 *p_ub, kmp_int32 *p_st);3827extern int __kmpc_dispatch_next_8(ident_t *loc, kmp_int32 gtid,3828 kmp_int32 *p_last, kmp_int64 *p_lb,3829 kmp_int64 *p_ub, kmp_int64 *p_st);3830extern int __kmpc_dispatch_next_8u(ident_t *loc, kmp_int32 gtid,3831 kmp_int32 *p_last, kmp_uint64 *p_lb,3832 kmp_uint64 *p_ub, kmp_int64 *p_st);3833 3834extern void __kmpc_dispatch_fini_4(ident_t *loc, kmp_int32 gtid);3835extern void __kmpc_dispatch_fini_8(ident_t *loc, kmp_int32 gtid);3836extern void __kmpc_dispatch_fini_4u(ident_t *loc, kmp_int32 gtid);3837extern void __kmpc_dispatch_fini_8u(ident_t *loc, kmp_int32 gtid);3838 3839extern void __kmpc_dispatch_deinit(ident_t *loc, kmp_int32 gtid);3840 3841#ifdef KMP_GOMP_COMPAT3842 3843extern void __kmp_aux_dispatch_init_4(ident_t *loc, kmp_int32 gtid,3844 enum sched_type schedule, kmp_int32 lb,3845 kmp_int32 ub, kmp_int32 st,3846 kmp_int32 chunk, int push_ws);3847extern void __kmp_aux_dispatch_init_4u(ident_t *loc, kmp_int32 gtid,3848 enum sched_type schedule, kmp_uint32 lb,3849 kmp_uint32 ub, kmp_int32 st,3850 kmp_int32 chunk, int push_ws);3851extern void __kmp_aux_dispatch_init_8(ident_t *loc, kmp_int32 gtid,3852 enum sched_type schedule, kmp_int64 lb,3853 kmp_int64 ub, kmp_int64 st,3854 kmp_int64 chunk, int push_ws);3855extern void __kmp_aux_dispatch_init_8u(ident_t *loc, kmp_int32 gtid,3856 enum sched_type schedule, kmp_uint64 lb,3857 kmp_uint64 ub, kmp_int64 st,3858 kmp_int64 chunk, int push_ws);3859extern void __kmp_aux_dispatch_fini_chunk_4(ident_t *loc, kmp_int32 gtid);3860extern void __kmp_aux_dispatch_fini_chunk_8(ident_t *loc, kmp_int32 gtid);3861extern void __kmp_aux_dispatch_fini_chunk_4u(ident_t *loc, kmp_int32 gtid);3862extern void __kmp_aux_dispatch_fini_chunk_8u(ident_t *loc, kmp_int32 gtid);3863 3864#endif /* KMP_GOMP_COMPAT */3865 3866extern kmp_uint32 __kmp_eq_4(kmp_uint32 value, kmp_uint32 checker);3867extern kmp_uint32 __kmp_neq_4(kmp_uint32 value, kmp_uint32 checker);3868extern kmp_uint32 __kmp_lt_4(kmp_uint32 value, kmp_uint32 checker);3869extern kmp_uint32 __kmp_ge_4(kmp_uint32 value, kmp_uint32 checker);3870extern kmp_uint32 __kmp_le_4(kmp_uint32 value, kmp_uint32 checker);3871extern kmp_uint32 __kmp_wait_4(kmp_uint32 volatile *spinner, kmp_uint32 checker,3872 kmp_uint32 (*pred)(kmp_uint32, kmp_uint32),3873 void *obj);3874extern void __kmp_wait_4_ptr(void *spinner, kmp_uint32 checker,3875 kmp_uint32 (*pred)(void *, kmp_uint32), void *obj);3876 3877extern void __kmp_wait_64(kmp_info_t *this_thr, kmp_flag_64<> *flag,3878 int final_spin3879#if USE_ITT_BUILD3880 ,3881 void *itt_sync_obj3882#endif3883);3884extern void __kmp_release_64(kmp_flag_64<> *flag);3885 3886extern void __kmp_infinite_loop(void);3887 3888extern void __kmp_cleanup(void);3889 3890#if KMP_HANDLE_SIGNALS3891extern int __kmp_handle_signals;3892extern void __kmp_install_signals(int parallel_init);3893extern void __kmp_remove_signals(void);3894#endif3895 3896extern void __kmp_clear_system_time(void);3897extern void __kmp_read_system_time(double *delta);3898 3899extern void __kmp_check_stack_overlap(kmp_info_t *thr);3900 3901extern void __kmp_expand_host_name(char *buffer, size_t size);3902extern void __kmp_expand_file_name(char *result, size_t rlen, char *pattern);3903 3904#if KMP_ARCH_X86 || KMP_ARCH_X86_64 || (KMP_OS_WINDOWS && (KMP_ARCH_AARCH64 || KMP_ARCH_ARM))3905extern void3906__kmp_initialize_system_tick(void); /* Initialize timer tick value */3907#endif3908 3909extern void3910__kmp_runtime_initialize(void); /* machine specific initialization */3911extern void __kmp_runtime_destroy(void);3912 3913#if KMP_AFFINITY_SUPPORTED3914extern char *__kmp_affinity_print_mask(char *buf, int buf_len,3915 kmp_affin_mask_t *mask);3916extern kmp_str_buf_t *__kmp_affinity_str_buf_mask(kmp_str_buf_t *buf,3917 kmp_affin_mask_t *mask);3918extern void __kmp_affinity_initialize(kmp_affinity_t &affinity);3919extern void __kmp_affinity_uninitialize(void);3920extern void __kmp_affinity_set_init_mask(3921 int gtid, int isa_root); /* set affinity according to KMP_AFFINITY */3922void __kmp_affinity_bind_init_mask(int gtid);3923extern void __kmp_affinity_bind_place(int gtid);3924extern void __kmp_affinity_determine_capable(const char *env_var);3925extern int __kmp_aux_set_affinity(void **mask);3926extern int __kmp_aux_get_affinity(void **mask);3927extern int __kmp_aux_get_affinity_max_proc();3928extern int __kmp_aux_set_affinity_mask_proc(int proc, void **mask);3929extern int __kmp_aux_unset_affinity_mask_proc(int proc, void **mask);3930extern int __kmp_aux_get_affinity_mask_proc(int proc, void **mask);3931extern void __kmp_balanced_affinity(kmp_info_t *th, int team_size);3932#if KMP_WEIGHTED_ITERATIONS_SUPPORTED3933extern int __kmp_get_first_osid_with_ecore(void);3934#endif3935#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \3936 KMP_OS_AIX3937extern int kmp_set_thread_affinity_mask_initial(void);3938#endif3939static inline void __kmp_assign_root_init_mask() {3940 int gtid = __kmp_entry_gtid();3941 kmp_root_t *r = __kmp_threads[gtid]->th.th_root;3942 if (r->r.r_uber_thread == __kmp_threads[gtid] && !r->r.r_affinity_assigned) {3943 __kmp_affinity_set_init_mask(gtid, /*isa_root=*/TRUE);3944 __kmp_affinity_bind_init_mask(gtid);3945 r->r.r_affinity_assigned = TRUE;3946 }3947}3948static inline void __kmp_reset_root_init_mask(int gtid) {3949 if (!KMP_AFFINITY_CAPABLE())3950 return;3951 kmp_info_t *th = __kmp_threads[gtid];3952 kmp_root_t *r = th->th.th_root;3953 if (r->r.r_uber_thread == th && r->r.r_affinity_assigned) {3954 __kmp_set_system_affinity(__kmp_affin_origMask, FALSE);3955 KMP_CPU_COPY(th->th.th_affin_mask, __kmp_affin_origMask);3956 r->r.r_affinity_assigned = FALSE;3957 }3958}3959#else /* KMP_AFFINITY_SUPPORTED */3960#define __kmp_assign_root_init_mask() /* Nothing */3961static inline void __kmp_reset_root_init_mask(int gtid) {}3962#endif /* KMP_AFFINITY_SUPPORTED */3963// No need for KMP_AFFINITY_SUPPORTED guard as only one field in the3964// format string is for affinity, so platforms that do not support3965// affinity can still use the other fields, e.g., %n for num_threads3966extern size_t __kmp_aux_capture_affinity(int gtid, const char *format,3967 kmp_str_buf_t *buffer);3968extern void __kmp_aux_display_affinity(int gtid, const char *format);3969 3970extern void __kmp_cleanup_hierarchy();3971extern void __kmp_get_hierarchy(kmp_uint32 nproc, kmp_bstate_t *thr_bar);3972 3973#if KMP_USE_FUTEX3974 3975extern int __kmp_futex_determine_capable(void);3976 3977#endif // KMP_USE_FUTEX3978 3979extern void __kmp_gtid_set_specific(int gtid);3980extern int __kmp_gtid_get_specific(void);3981 3982extern double __kmp_read_cpu_time(void);3983 3984extern int __kmp_read_system_info(struct kmp_sys_info *info);3985 3986#if KMP_USE_MONITOR3987extern void __kmp_create_monitor(kmp_info_t *th);3988#endif3989 3990extern void *__kmp_launch_thread(kmp_info_t *thr);3991 3992extern void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size);3993 3994#if KMP_OS_WINDOWS3995extern int __kmp_still_running(kmp_info_t *th);3996extern int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val);3997extern void __kmp_free_handle(kmp_thread_t tHandle);3998#endif3999 4000#if KMP_USE_MONITOR4001extern void __kmp_reap_monitor(kmp_info_t *th);4002#endif4003extern void __kmp_reap_worker(kmp_info_t *th);4004extern void __kmp_terminate_thread(int gtid);4005 4006extern int __kmp_try_suspend_mx(kmp_info_t *th);4007extern void __kmp_lock_suspend_mx(kmp_info_t *th);4008extern void __kmp_unlock_suspend_mx(kmp_info_t *th);4009 4010extern void __kmp_elapsed(double *);4011extern void __kmp_elapsed_tick(double *);4012 4013extern void __kmp_enable(int old_state);4014extern void __kmp_disable(int *old_state);4015 4016extern void __kmp_thread_sleep(int millis);4017 4018extern void __kmp_common_initialize(void);4019extern void __kmp_common_destroy(void);4020extern void __kmp_common_destroy_gtid(int gtid);4021 4022#if KMP_OS_UNIX4023extern void __kmp_register_atfork(void);4024#endif4025extern void __kmp_suspend_initialize(void);4026extern void __kmp_suspend_initialize_thread(kmp_info_t *th);4027extern void __kmp_suspend_uninitialize_thread(kmp_info_t *th);4028 4029extern kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,4030 int tid);4031extern kmp_team_t *__kmp_allocate_team(kmp_root_t *root, int new_nproc,4032 int max_nproc,4033#if OMPT_SUPPORT4034 ompt_data_t ompt_parallel_data,4035#endif4036 kmp_proc_bind_t proc_bind,4037 kmp_internal_control_t *new_icvs,4038 int argc, kmp_info_t *thr);4039extern void __kmp_free_thread(kmp_info_t *);4040extern void __kmp_free_team(kmp_root_t *, kmp_team_t *, kmp_info_t *);4041extern kmp_team_t *__kmp_reap_team(kmp_team_t *);4042 4043/* ------------------------------------------------------------------------ */4044 4045extern void __kmp_initialize_bget(kmp_info_t *th);4046extern void __kmp_finalize_bget(kmp_info_t *th);4047 4048KMP_EXPORT void *kmpc_malloc(size_t size);4049KMP_EXPORT void *kmpc_aligned_malloc(size_t size, size_t alignment);4050KMP_EXPORT void *kmpc_calloc(size_t nelem, size_t elsize);4051KMP_EXPORT void *kmpc_realloc(void *ptr, size_t size);4052KMP_EXPORT void kmpc_free(void *ptr);4053 4054/* declarations for internal use */4055 4056extern int __kmp_barrier(enum barrier_type bt, int gtid, int is_split,4057 size_t reduce_size, void *reduce_data,4058 void (*reduce)(void *, void *));4059extern void __kmp_end_split_barrier(enum barrier_type bt, int gtid);4060extern int __kmp_barrier_gomp_cancel(int gtid);4061 4062/*!4063 * Tell the fork call which compiler generated the fork call, and therefore how4064 * to deal with the call.4065 */4066enum fork_context_e {4067 fork_context_gnu, /**< Called from GNU generated code, so must not invoke the4068 microtask internally. */4069 fork_context_intel, /**< Called from Intel generated code. */4070 fork_context_last4071};4072extern int __kmp_fork_call(ident_t *loc, int gtid,4073 enum fork_context_e fork_context, kmp_int32 argc,4074 microtask_t microtask, launch_t invoker,4075 kmp_va_list ap);4076 4077extern void __kmp_join_call(ident_t *loc, int gtid4078#if OMPT_SUPPORT4079 ,4080 enum fork_context_e fork_context4081#endif4082 ,4083 int exit_teams = 0);4084 4085extern void __kmp_serialized_parallel(ident_t *id, kmp_int32 gtid);4086extern void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team);4087extern void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team);4088extern int __kmp_invoke_task_func(int gtid);4089extern void __kmp_run_before_invoked_task(int gtid, int tid,4090 kmp_info_t *this_thr,4091 kmp_team_t *team);4092extern void __kmp_run_after_invoked_task(int gtid, int tid,4093 kmp_info_t *this_thr,4094 kmp_team_t *team);4095 4096// should never have been exported4097KMP_EXPORT int __kmpc_invoke_task_func(int gtid);4098extern int __kmp_invoke_teams_master(int gtid);4099extern void __kmp_teams_master(int gtid);4100extern int __kmp_aux_get_team_num();4101extern int __kmp_aux_get_num_teams();4102extern void __kmp_save_internal_controls(kmp_info_t *thread);4103extern void __kmp_user_set_library(enum library_type arg);4104extern void __kmp_aux_set_library(enum library_type arg);4105extern void __kmp_aux_set_stacksize(size_t arg);4106extern void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid);4107extern void __kmp_aux_set_defaults(char const *str, size_t len);4108 4109/* Functions called from __kmp_aux_env_initialize() in kmp_settings.cpp */4110void kmpc_set_blocktime(int arg);4111void ompc_set_nested(int flag);4112void ompc_set_dynamic(int flag);4113void ompc_set_num_threads(int arg);4114 4115extern void __kmp_push_current_task_to_thread(kmp_info_t *this_thr,4116 kmp_team_t *team, int tid);4117extern void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr);4118extern kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,4119 kmp_tasking_flags_t *flags,4120 size_t sizeof_kmp_task_t,4121 size_t sizeof_shareds,4122 kmp_routine_entry_t task_entry);4123extern void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr,4124 kmp_team_t *team, int tid,4125 int set_curr_task);4126extern void __kmp_finish_implicit_task(kmp_info_t *this_thr);4127extern void __kmp_free_implicit_task(kmp_info_t *this_thr);4128 4129extern kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref,4130 int gtid,4131 kmp_task_t *task);4132extern void __kmp_fulfill_event(kmp_event_t *event);4133 4134extern void __kmp_free_task_team(kmp_info_t *thread,4135 kmp_task_team_t *task_team);4136extern void __kmp_reap_task_teams(void);4137extern void __kmp_push_task_team_node(kmp_info_t *thread, kmp_team_t *team);4138extern void __kmp_pop_task_team_node(kmp_info_t *thread, kmp_team_t *team);4139extern void __kmp_wait_to_unref_task_teams(void);4140extern void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team);4141extern void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team);4142extern void __kmp_task_team_wait(kmp_info_t *this_thr, kmp_team_t *team4143#if USE_ITT_BUILD4144 ,4145 void *itt_sync_obj4146#endif /* USE_ITT_BUILD */4147 ,4148 int wait = 1);4149extern void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread,4150 int gtid);4151#if KMP_DEBUG4152#define KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, thr) \4153 KMP_DEBUG_ASSERT( \4154 __kmp_tasking_mode != tskm_task_teams || team->t.t_nproc == 1 || \4155 thr->th.th_task_team == team->t.t_task_team[thr->th.th_task_state])4156#else4157#define KMP_DEBUG_ASSERT_TASKTEAM_INVARIANT(team, thr) /* Nothing */4158#endif4159 4160extern int __kmp_is_address_mapped(void *addr);4161extern kmp_uint64 __kmp_hardware_timestamp(void);4162 4163#if KMP_OS_UNIX4164extern int __kmp_read_from_file(char const *path, char const *format, ...);4165#endif4166 4167/* ------------------------------------------------------------------------ */4168//4169// Assembly routines that have no compiler intrinsic replacement4170//4171 4172extern int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int npr, int argc,4173 void *argv[]4174#if OMPT_SUPPORT4175 ,4176 void **exit_frame_ptr4177#endif4178);4179 4180/* ------------------------------------------------------------------------ */4181 4182KMP_EXPORT void __kmpc_begin(ident_t *, kmp_int32 flags);4183KMP_EXPORT void __kmpc_end(ident_t *);4184 4185KMP_EXPORT void __kmpc_threadprivate_register_vec(ident_t *, void *data,4186 kmpc_ctor_vec ctor,4187 kmpc_cctor_vec cctor,4188 kmpc_dtor_vec dtor,4189 size_t vector_length);4190KMP_EXPORT void __kmpc_threadprivate_register(ident_t *, void *data,4191 kmpc_ctor ctor, kmpc_cctor cctor,4192 kmpc_dtor dtor);4193KMP_EXPORT void *__kmpc_threadprivate(ident_t *, kmp_int32 global_tid,4194 void *data, size_t size);4195 4196KMP_EXPORT kmp_int32 __kmpc_global_thread_num(ident_t *);4197KMP_EXPORT kmp_int32 __kmpc_global_num_threads(ident_t *);4198KMP_EXPORT kmp_int32 __kmpc_bound_thread_num(ident_t *);4199KMP_EXPORT kmp_int32 __kmpc_bound_num_threads(ident_t *);4200 4201KMP_EXPORT kmp_int32 __kmpc_ok_to_fork(ident_t *);4202KMP_EXPORT void __kmpc_fork_call(ident_t *, kmp_int32 nargs,4203 kmpc_micro microtask, ...);4204KMP_EXPORT void __kmpc_fork_call_if(ident_t *loc, kmp_int32 nargs,4205 kmpc_micro microtask, kmp_int32 cond,4206 void *args);4207 4208KMP_EXPORT void __kmpc_serialized_parallel(ident_t *, kmp_int32 global_tid);4209KMP_EXPORT void __kmpc_end_serialized_parallel(ident_t *, kmp_int32 global_tid);4210 4211KMP_EXPORT void __kmpc_flush(ident_t *);4212KMP_EXPORT void __kmpc_barrier(ident_t *, kmp_int32 global_tid);4213KMP_EXPORT kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid);4214KMP_EXPORT void __kmpc_end_master(ident_t *, kmp_int32 global_tid);4215KMP_EXPORT kmp_int32 __kmpc_masked(ident_t *, kmp_int32 global_tid,4216 kmp_int32 filter);4217KMP_EXPORT void __kmpc_end_masked(ident_t *, kmp_int32 global_tid);4218KMP_EXPORT void __kmpc_ordered(ident_t *, kmp_int32 global_tid);4219KMP_EXPORT void __kmpc_end_ordered(ident_t *, kmp_int32 global_tid);4220KMP_EXPORT void __kmpc_critical(ident_t *, kmp_int32 global_tid,4221 kmp_critical_name *);4222KMP_EXPORT void __kmpc_end_critical(ident_t *, kmp_int32 global_tid,4223 kmp_critical_name *);4224KMP_EXPORT void __kmpc_critical_with_hint(ident_t *, kmp_int32 global_tid,4225 kmp_critical_name *, uint32_t hint);4226 4227KMP_EXPORT kmp_int32 __kmpc_barrier_master(ident_t *, kmp_int32 global_tid);4228KMP_EXPORT void __kmpc_end_barrier_master(ident_t *, kmp_int32 global_tid);4229 4230KMP_EXPORT kmp_int32 __kmpc_barrier_master_nowait(ident_t *,4231 kmp_int32 global_tid);4232 4233KMP_EXPORT kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid);4234KMP_EXPORT void __kmpc_end_single(ident_t *, kmp_int32 global_tid);4235 4236KMP_EXPORT kmp_int32 __kmpc_sections_init(ident_t *loc, kmp_int32 global_tid);4237KMP_EXPORT kmp_int32 __kmpc_next_section(ident_t *loc, kmp_int32 global_tid,4238 kmp_int32 numberOfSections);4239KMP_EXPORT void __kmpc_end_sections(ident_t *loc, kmp_int32 global_tid);4240 4241KMP_EXPORT void KMPC_FOR_STATIC_INIT(ident_t *loc, kmp_int32 global_tid,4242 kmp_int32 schedtype, kmp_int32 *plastiter,4243 kmp_int *plower, kmp_int *pupper,4244 kmp_int *pstride, kmp_int incr,4245 kmp_int chunk);4246 4247KMP_EXPORT void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid);4248 4249KMP_EXPORT void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid,4250 size_t cpy_size, void *cpy_data,4251 void (*cpy_func)(void *, void *),4252 kmp_int32 didit);4253 4254KMP_EXPORT void *__kmpc_copyprivate_light(ident_t *loc, kmp_int32 gtid,4255 void *cpy_data);4256 4257extern void KMPC_SET_NUM_THREADS(int arg);4258extern void KMPC_SET_DYNAMIC(int flag);4259extern void KMPC_SET_NESTED(int flag);4260 4261/* OMP 3.0 tasking interface routines */4262KMP_EXPORT kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid,4263 kmp_task_t *new_task);4264KMP_EXPORT kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,4265 kmp_int32 flags,4266 size_t sizeof_kmp_task_t,4267 size_t sizeof_shareds,4268 kmp_routine_entry_t task_entry);4269KMP_EXPORT kmp_task_t *__kmpc_omp_target_task_alloc(4270 ident_t *loc_ref, kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t,4271 size_t sizeof_shareds, kmp_routine_entry_t task_entry, kmp_int64 device_id);4272KMP_EXPORT void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid,4273 kmp_task_t *task);4274KMP_EXPORT void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid,4275 kmp_task_t *task);4276KMP_EXPORT kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid,4277 kmp_task_t *new_task);4278KMP_EXPORT kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid);4279KMP_EXPORT kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid,4280 int end_part);4281 4282#if TASK_UNUSED4283void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task);4284void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid,4285 kmp_task_t *task);4286#endif // TASK_UNUSED4287 4288/* ------------------------------------------------------------------------ */4289 4290KMP_EXPORT void __kmpc_taskgroup(ident_t *loc, int gtid);4291KMP_EXPORT void __kmpc_end_taskgroup(ident_t *loc, int gtid);4292 4293KMP_EXPORT kmp_int32 __kmpc_omp_task_with_deps(4294 ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 ndeps,4295 kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias,4296 kmp_depend_info_t *noalias_dep_list);4297 4298KMP_EXPORT kmp_base_depnode_t *__kmpc_task_get_depnode(kmp_task_t *task);4299 4300KMP_EXPORT kmp_depnode_list_t *__kmpc_task_get_successors(kmp_task_t *task);4301 4302KMP_EXPORT void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid,4303 kmp_int32 ndeps,4304 kmp_depend_info_t *dep_list,4305 kmp_int32 ndeps_noalias,4306 kmp_depend_info_t *noalias_dep_list);4307/* __kmpc_omp_taskwait_deps_51 : Function for OpenMP 5.1 nowait clause.4308 * Placeholder for taskwait with nowait clause.*/4309KMP_EXPORT void __kmpc_omp_taskwait_deps_51(ident_t *loc_ref, kmp_int32 gtid,4310 kmp_int32 ndeps,4311 kmp_depend_info_t *dep_list,4312 kmp_int32 ndeps_noalias,4313 kmp_depend_info_t *noalias_dep_list,4314 kmp_int32 has_no_wait);4315 4316extern kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,4317 bool serialize_immediate);4318 4319KMP_EXPORT kmp_int32 __kmpc_cancel(ident_t *loc_ref, kmp_int32 gtid,4320 kmp_int32 cncl_kind);4321KMP_EXPORT kmp_int32 __kmpc_cancellationpoint(ident_t *loc_ref, kmp_int32 gtid,4322 kmp_int32 cncl_kind);4323KMP_EXPORT kmp_int32 __kmpc_cancel_barrier(ident_t *loc_ref, kmp_int32 gtid);4324KMP_EXPORT int __kmp_get_cancellation_status(int cancel_kind);4325 4326KMP_EXPORT void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask);4327KMP_EXPORT void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask);4328KMP_EXPORT void __kmpc_taskloop(ident_t *loc, kmp_int32 gtid, kmp_task_t *task,4329 kmp_int32 if_val, kmp_uint64 *lb,4330 kmp_uint64 *ub, kmp_int64 st, kmp_int32 nogroup,4331 kmp_int32 sched, kmp_uint64 grainsize,4332 void *task_dup);4333KMP_EXPORT void __kmpc_taskloop_5(ident_t *loc, kmp_int32 gtid,4334 kmp_task_t *task, kmp_int32 if_val,4335 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,4336 kmp_int32 nogroup, kmp_int32 sched,4337 kmp_uint64 grainsize, kmp_int32 modifier,4338 void *task_dup);4339KMP_EXPORT void *__kmpc_task_reduction_init(int gtid, int num_data, void *data);4340KMP_EXPORT void *__kmpc_taskred_init(int gtid, int num_data, void *data);4341KMP_EXPORT void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void *d);4342KMP_EXPORT void *__kmpc_task_reduction_modifier_init(ident_t *loc, int gtid,4343 int is_ws, int num,4344 void *data);4345KMP_EXPORT void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws,4346 int num, void *data);4347KMP_EXPORT void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid,4348 int is_ws);4349KMP_EXPORT kmp_int32 __kmpc_omp_reg_task_with_affinity(4350 ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 naffins,4351 kmp_task_affinity_info_t *affin_list);4352KMP_EXPORT void __kmp_set_num_teams(int num_teams);4353KMP_EXPORT int __kmp_get_max_teams(void);4354KMP_EXPORT void __kmp_set_teams_thread_limit(int limit);4355KMP_EXPORT int __kmp_get_teams_thread_limit(void);4356 4357/* Interface target task integration */4358KMP_EXPORT void **__kmpc_omp_get_target_async_handle_ptr(kmp_int32 gtid);4359KMP_EXPORT bool __kmpc_omp_has_task_team(kmp_int32 gtid);4360 4361/* Lock interface routines (fast versions with gtid passed in) */4362KMP_EXPORT void __kmpc_init_lock(ident_t *loc, kmp_int32 gtid,4363 void **user_lock);4364KMP_EXPORT void __kmpc_init_nest_lock(ident_t *loc, kmp_int32 gtid,4365 void **user_lock);4366KMP_EXPORT void __kmpc_destroy_lock(ident_t *loc, kmp_int32 gtid,4367 void **user_lock);4368KMP_EXPORT void __kmpc_destroy_nest_lock(ident_t *loc, kmp_int32 gtid,4369 void **user_lock);4370KMP_EXPORT void __kmpc_set_lock(ident_t *loc, kmp_int32 gtid, void **user_lock);4371KMP_EXPORT void __kmpc_set_nest_lock(ident_t *loc, kmp_int32 gtid,4372 void **user_lock);4373KMP_EXPORT void __kmpc_unset_lock(ident_t *loc, kmp_int32 gtid,4374 void **user_lock);4375KMP_EXPORT void __kmpc_unset_nest_lock(ident_t *loc, kmp_int32 gtid,4376 void **user_lock);4377KMP_EXPORT int __kmpc_test_lock(ident_t *loc, kmp_int32 gtid, void **user_lock);4378KMP_EXPORT int __kmpc_test_nest_lock(ident_t *loc, kmp_int32 gtid,4379 void **user_lock);4380 4381KMP_EXPORT void __kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid,4382 void **user_lock, uintptr_t hint);4383KMP_EXPORT void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid,4384 void **user_lock,4385 uintptr_t hint);4386 4387#if OMPX_TASKGRAPH4388// Taskgraph's Record & Replay mechanism4389// __kmp_tdg_is_recording: check whether a given TDG is recording4390// status: the tdg's current status4391static inline bool __kmp_tdg_is_recording(kmp_tdg_status_t status) {4392 return status == KMP_TDG_RECORDING;4393}4394 4395KMP_EXPORT kmp_int32 __kmpc_start_record_task(ident_t *loc, kmp_int32 gtid,4396 kmp_int32 input_flags,4397 kmp_int32 tdg_id);4398KMP_EXPORT void __kmpc_end_record_task(ident_t *loc, kmp_int32 gtid,4399 kmp_int32 input_flags, kmp_int32 tdg_id);4400#endif4401/* Interface to fast scalable reduce methods routines */4402 4403KMP_EXPORT kmp_int32 __kmpc_reduce_nowait(4404 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,4405 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),4406 kmp_critical_name *lck);4407KMP_EXPORT void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,4408 kmp_critical_name *lck);4409KMP_EXPORT kmp_int32 __kmpc_reduce(4410 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,4411 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),4412 kmp_critical_name *lck);4413KMP_EXPORT void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,4414 kmp_critical_name *lck);4415 4416/* Internal fast reduction routines */4417 4418extern PACKED_REDUCTION_METHOD_T __kmp_determine_reduction_method(4419 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,4420 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),4421 kmp_critical_name *lck);4422 4423// this function is for testing set/get/determine reduce method4424KMP_EXPORT kmp_int32 __kmp_get_reduce_method(void);4425 4426KMP_EXPORT kmp_uint64 __kmpc_get_taskid();4427KMP_EXPORT kmp_uint64 __kmpc_get_parent_taskid();4428 4429// C++ port4430// missing 'extern "C"' declarations4431 4432KMP_EXPORT kmp_int32 __kmpc_in_parallel(ident_t *loc);4433KMP_EXPORT void __kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid);4434KMP_EXPORT void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,4435 kmp_int32 num_threads);4436KMP_EXPORT void __kmpc_push_num_threads_strict(ident_t *loc,4437 kmp_int32 global_tid,4438 kmp_int32 num_threads,4439 int severity,4440 const char *message);4441 4442KMP_EXPORT void __kmpc_push_num_threads_list(ident_t *loc, kmp_int32 global_tid,4443 kmp_uint32 list_length,4444 kmp_int32 *num_threads_list);4445KMP_EXPORT void __kmpc_push_num_threads_list_strict(4446 ident_t *loc, kmp_int32 global_tid, kmp_uint32 list_length,4447 kmp_int32 *num_threads_list, int severity, const char *message);4448 4449KMP_EXPORT void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,4450 int proc_bind);4451KMP_EXPORT void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,4452 kmp_int32 num_teams,4453 kmp_int32 num_threads);4454KMP_EXPORT void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid,4455 kmp_int32 thread_limit);4456/* Function for OpenMP 5.1 num_teams clause */4457KMP_EXPORT void __kmpc_push_num_teams_51(ident_t *loc, kmp_int32 global_tid,4458 kmp_int32 num_teams_lb,4459 kmp_int32 num_teams_ub,4460 kmp_int32 num_threads);4461KMP_EXPORT void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc,4462 kmpc_micro microtask, ...);4463struct kmp_dim { // loop bounds info casted to kmp_int644464 kmp_int64 lo; // lower4465 kmp_int64 up; // upper4466 kmp_int64 st; // stride4467};4468KMP_EXPORT void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid,4469 kmp_int32 num_dims,4470 const struct kmp_dim *dims);4471KMP_EXPORT void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid,4472 const kmp_int64 *vec);4473KMP_EXPORT void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid,4474 const kmp_int64 *vec);4475KMP_EXPORT void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid);4476 4477KMP_EXPORT void *__kmpc_threadprivate_cached(ident_t *loc, kmp_int32 global_tid,4478 void *data, size_t size,4479 void ***cache);4480 4481// The routines below are not exported.4482// Consider making them 'static' in corresponding source files.4483void kmp_threadprivate_insert_private_data(int gtid, void *pc_addr,4484 void *data_addr, size_t pc_size);4485struct private_common *kmp_threadprivate_insert(int gtid, void *pc_addr,4486 void *data_addr,4487 size_t pc_size);4488void __kmp_threadprivate_resize_cache(int newCapacity);4489void __kmp_cleanup_threadprivate_caches();4490 4491// ompc_, kmpc_ entries moved from omp.h.4492#if KMP_OS_WINDOWS4493#define KMPC_CONVENTION __cdecl4494#else4495#define KMPC_CONVENTION4496#endif4497 4498#ifndef __OMP_H4499typedef enum omp_sched_t {4500 omp_sched_static = 1,4501 omp_sched_dynamic = 2,4502 omp_sched_guided = 3,4503 omp_sched_auto = 44504} omp_sched_t;4505typedef void *kmp_affinity_mask_t;4506#endif4507 4508KMP_EXPORT void KMPC_CONVENTION ompc_set_max_active_levels(int);4509KMP_EXPORT void KMPC_CONVENTION ompc_set_schedule(omp_sched_t, int);4510KMP_EXPORT int KMPC_CONVENTION ompc_get_ancestor_thread_num(int);4511KMP_EXPORT int KMPC_CONVENTION ompc_get_team_size(int);4512KMP_EXPORT int KMPC_CONVENTION4513kmpc_set_affinity_mask_proc(int, kmp_affinity_mask_t *);4514KMP_EXPORT int KMPC_CONVENTION4515kmpc_unset_affinity_mask_proc(int, kmp_affinity_mask_t *);4516KMP_EXPORT int KMPC_CONVENTION4517kmpc_get_affinity_mask_proc(int, kmp_affinity_mask_t *);4518 4519KMP_EXPORT void KMPC_CONVENTION kmpc_set_stacksize(int);4520KMP_EXPORT void KMPC_CONVENTION kmpc_set_stacksize_s(size_t);4521KMP_EXPORT void KMPC_CONVENTION kmpc_set_library(int);4522KMP_EXPORT void KMPC_CONVENTION kmpc_set_defaults(char const *);4523KMP_EXPORT void KMPC_CONVENTION kmpc_set_disp_num_buffers(int);4524void KMP_EXPAND_NAME(ompc_set_affinity_format)(char const *format);4525size_t KMP_EXPAND_NAME(ompc_get_affinity_format)(char *buffer, size_t size);4526void KMP_EXPAND_NAME(ompc_display_affinity)(char const *format);4527size_t KMP_EXPAND_NAME(ompc_capture_affinity)(char *buffer, size_t buf_size,4528 char const *format);4529 4530enum kmp_target_offload_kind {4531 tgt_disabled = 0,4532 tgt_default = 1,4533 tgt_mandatory = 24534};4535typedef enum kmp_target_offload_kind kmp_target_offload_kind_t;4536// Set via OMP_TARGET_OFFLOAD if specified, defaults to tgt_default otherwise4537extern kmp_target_offload_kind_t __kmp_target_offload;4538extern int __kmpc_get_target_offload();4539 4540// Constants used in libomptarget4541#define KMP_DEVICE_DEFAULT -1 // This is libomptarget's default device.4542#define KMP_DEVICE_ALL -11 // This is libomptarget's "all devices".4543 4544// OMP Pause Resource4545 4546// The following enum is used both to set the status in __kmp_pause_status, and4547// as the internal equivalent of the externally-visible omp_pause_resource_t.4548typedef enum kmp_pause_status_t {4549 kmp_not_paused = 0, // status is not paused, or, requesting resume4550 kmp_soft_paused = 1, // status is soft-paused, or, requesting soft pause4551 kmp_hard_paused = 2, // status is hard-paused, or, requesting hard pause4552 kmp_stop_tool_paused = 3 // requesting stop_tool pause4553} kmp_pause_status_t;4554 4555// This stores the pause state of the runtime4556extern kmp_pause_status_t __kmp_pause_status;4557extern int __kmpc_pause_resource(kmp_pause_status_t level);4558extern int __kmp_pause_resource(kmp_pause_status_t level);4559// Soft resume sets __kmp_pause_status, and wakes up all threads.4560extern void __kmp_resume_if_soft_paused();4561// Hard resume simply resets the status to not paused. Library will appear to4562// be uninitialized after hard pause. Let OMP constructs trigger required4563// initializations.4564static inline void __kmp_resume_if_hard_paused() {4565 if (__kmp_pause_status == kmp_hard_paused) {4566 __kmp_pause_status = kmp_not_paused;4567 }4568}4569 4570extern void __kmp_omp_display_env(int verbose);4571 4572// 1: it is initializing hidden helper team4573extern volatile int __kmp_init_hidden_helper;4574// 1: the hidden helper team is done4575extern volatile int __kmp_hidden_helper_team_done;4576// 1: enable hidden helper task4577extern kmp_int32 __kmp_enable_hidden_helper;4578// Main thread of hidden helper team4579extern kmp_info_t *__kmp_hidden_helper_main_thread;4580// Descriptors for the hidden helper threads4581extern kmp_info_t **__kmp_hidden_helper_threads;4582// Number of hidden helper threads4583extern kmp_int32 __kmp_hidden_helper_threads_num;4584// Number of hidden helper tasks that have not been executed yet4585extern std::atomic<kmp_int32> __kmp_unexecuted_hidden_helper_tasks;4586 4587extern void __kmp_hidden_helper_initialize();4588extern void __kmp_hidden_helper_threads_initz_routine();4589extern void __kmp_do_initialize_hidden_helper_threads();4590extern void __kmp_hidden_helper_threads_initz_wait();4591extern void __kmp_hidden_helper_initz_release();4592extern void __kmp_hidden_helper_threads_deinitz_wait();4593extern void __kmp_hidden_helper_threads_deinitz_release();4594extern void __kmp_hidden_helper_main_thread_wait();4595extern void __kmp_hidden_helper_worker_thread_wait();4596extern void __kmp_hidden_helper_worker_thread_signal();4597extern void __kmp_hidden_helper_main_thread_release();4598 4599// Check whether a given thread is a hidden helper thread4600#define KMP_HIDDEN_HELPER_THREAD(gtid) \4601 ((gtid) >= 1 && (gtid) <= __kmp_hidden_helper_threads_num)4602 4603#define KMP_HIDDEN_HELPER_WORKER_THREAD(gtid) \4604 ((gtid) > 1 && (gtid) <= __kmp_hidden_helper_threads_num)4605 4606#define KMP_HIDDEN_HELPER_MAIN_THREAD(gtid) \4607 ((gtid) == 1 && (gtid) <= __kmp_hidden_helper_threads_num)4608 4609#define KMP_HIDDEN_HELPER_TEAM(team) \4610 (team->t.t_threads[0] == __kmp_hidden_helper_main_thread)4611 4612// Map a gtid to a hidden helper thread. The first hidden helper thread, a.k.a4613// main thread, is skipped.4614#define KMP_GTID_TO_SHADOW_GTID(gtid) \4615 ((gtid) % (__kmp_hidden_helper_threads_num - 1) + 2)4616 4617// Return the adjusted gtid value by subtracting from gtid the number4618// of hidden helper threads. This adjusted value is the gtid the thread would4619// have received if there were no hidden helper threads.4620static inline int __kmp_adjust_gtid_for_hidden_helpers(int gtid) {4621 int adjusted_gtid = gtid;4622 if (__kmp_hidden_helper_threads_num > 0 && gtid > 0 &&4623 gtid - __kmp_hidden_helper_threads_num >= 0) {4624 adjusted_gtid -= __kmp_hidden_helper_threads_num;4625 }4626 return adjusted_gtid;4627}4628 4629#if ENABLE_LIBOMPTARGET4630// Pointers to callbacks registered by the offload library to be notified of4631// task progress.4632extern void (*kmp_target_sync_cb)(ident_t *loc_ref, int gtid,4633 void *current_task, void *event);4634#endif // ENABLE_LIBOMPTARGET4635 4636// Support for error directive4637typedef enum kmp_severity_t {4638 severity_warning = 1,4639 severity_fatal = 24640} kmp_severity_t;4641extern void __kmpc_error(ident_t *loc, int severity, const char *message);4642 4643// Support for scope directive4644KMP_EXPORT void __kmpc_scope(ident_t *loc, kmp_int32 gtid, void *reserved);4645KMP_EXPORT void __kmpc_end_scope(ident_t *loc, kmp_int32 gtid, void *reserved);4646 4647#ifdef __cplusplus4648}4649#endif4650 4651template <bool C, bool S>4652extern void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag);4653template <bool C, bool S>4654extern void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag);4655template <bool C, bool S>4656extern void __kmp_atomic_suspend_64(int th_gtid,4657 kmp_atomic_flag_64<C, S> *flag);4658extern void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag);4659#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT4660template <bool C, bool S>4661extern void __kmp_mwait_32(int th_gtid, kmp_flag_32<C, S> *flag);4662template <bool C, bool S>4663extern void __kmp_mwait_64(int th_gtid, kmp_flag_64<C, S> *flag);4664template <bool C, bool S>4665extern void __kmp_atomic_mwait_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag);4666extern void __kmp_mwait_oncore(int th_gtid, kmp_flag_oncore *flag);4667#endif4668template <bool C, bool S>4669extern void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag);4670template <bool C, bool S>4671extern void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag);4672template <bool C, bool S>4673extern void __kmp_atomic_resume_64(int target_gtid,4674 kmp_atomic_flag_64<C, S> *flag);4675extern void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag);4676 4677template <bool C, bool S>4678int __kmp_execute_tasks_32(kmp_info_t *thread, kmp_int32 gtid,4679 kmp_flag_32<C, S> *flag, int final_spin,4680 int *thread_finished,4681#if USE_ITT_BUILD4682 void *itt_sync_obj,4683#endif /* USE_ITT_BUILD */4684 kmp_int32 is_constrained);4685template <bool C, bool S>4686int __kmp_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid,4687 kmp_flag_64<C, S> *flag, int final_spin,4688 int *thread_finished,4689#if USE_ITT_BUILD4690 void *itt_sync_obj,4691#endif /* USE_ITT_BUILD */4692 kmp_int32 is_constrained);4693template <bool C, bool S>4694int __kmp_atomic_execute_tasks_64(kmp_info_t *thread, kmp_int32 gtid,4695 kmp_atomic_flag_64<C, S> *flag,4696 int final_spin, int *thread_finished,4697#if USE_ITT_BUILD4698 void *itt_sync_obj,4699#endif /* USE_ITT_BUILD */4700 kmp_int32 is_constrained);4701int __kmp_execute_tasks_oncore(kmp_info_t *thread, kmp_int32 gtid,4702 kmp_flag_oncore *flag, int final_spin,4703 int *thread_finished,4704#if USE_ITT_BUILD4705 void *itt_sync_obj,4706#endif /* USE_ITT_BUILD */4707 kmp_int32 is_constrained);4708 4709extern int __kmp_nesting_mode;4710extern int __kmp_nesting_mode_nlevels;4711extern int *__kmp_nesting_nth_level;4712extern void __kmp_init_nesting_mode();4713extern void __kmp_set_nesting_mode_threads();4714 4715/// This class safely opens and closes a C-style FILE* object using RAII4716/// semantics. There are also methods which allow using stdout or stderr as4717/// the underlying FILE* object. With the implicit conversion operator to4718/// FILE*, an object with this type can be used in any function which takes4719/// a FILE* object e.g., fprintf().4720/// No close method is needed at use sites.4721class kmp_safe_raii_file_t {4722 FILE *f;4723 4724 void close() {4725 if (f && f != stdout && f != stderr) {4726 fclose(f);4727 f = nullptr;4728 }4729 }4730 4731public:4732 kmp_safe_raii_file_t() : f(nullptr) {}4733 kmp_safe_raii_file_t(const char *filename, const char *mode,4734 const char *env_var = nullptr)4735 : f(nullptr) {4736 open(filename, mode, env_var);4737 }4738 kmp_safe_raii_file_t(const kmp_safe_raii_file_t &other) = delete;4739 kmp_safe_raii_file_t &operator=(const kmp_safe_raii_file_t &other) = delete;4740 ~kmp_safe_raii_file_t() { close(); }4741 4742 /// Open filename using mode. This is automatically closed in the destructor.4743 /// The env_var parameter indicates the environment variable the filename4744 /// came from if != nullptr.4745 void open(const char *filename, const char *mode,4746 const char *env_var = nullptr) {4747 KMP_ASSERT(!f);4748 f = fopen(filename, mode);4749 if (!f) {4750 int code = errno;4751 if (env_var) {4752 __kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),4753 KMP_HNT(CheckEnvVar, env_var, filename), __kmp_msg_null);4754 } else {4755 __kmp_fatal(KMP_MSG(CantOpenFileForReading, filename), KMP_ERR(code),4756 __kmp_msg_null);4757 }4758 }4759 }4760 /// Instead of erroring out, return non-zero when4761 /// unsuccessful fopen() for any reason4762 int try_open(const char *filename, const char *mode) {4763 KMP_ASSERT(!f);4764 f = fopen(filename, mode);4765 if (!f)4766 return errno;4767 return 0;4768 }4769 /// Set the FILE* object to stdout and output there4770 /// No open call should happen before this call.4771 void set_stdout() {4772 KMP_ASSERT(!f);4773 f = stdout;4774 }4775 /// Set the FILE* object to stderr and output there4776 /// No open call should happen before this call.4777 void set_stderr() {4778 KMP_ASSERT(!f);4779 f = stderr;4780 }4781 operator bool() { return bool(f); }4782 operator FILE *() { return f; }4783};4784 4785template <typename SourceType, typename TargetType,4786 bool isSourceSmaller = (sizeof(SourceType) < sizeof(TargetType)),4787 bool isSourceEqual = (sizeof(SourceType) == sizeof(TargetType)),4788 bool isSourceSigned = std::is_signed<SourceType>::value,4789 bool isTargetSigned = std::is_signed<TargetType>::value>4790struct kmp_convert {};4791 4792// Both types are signed; Source smaller4793template <typename SourceType, typename TargetType>4794struct kmp_convert<SourceType, TargetType, true, false, true, true> {4795 static TargetType to(SourceType src) { return (TargetType)src; }4796};4797// Source equal4798template <typename SourceType, typename TargetType>4799struct kmp_convert<SourceType, TargetType, false, true, true, true> {4800 static TargetType to(SourceType src) { return src; }4801};4802// Source bigger4803template <typename SourceType, typename TargetType>4804struct kmp_convert<SourceType, TargetType, false, false, true, true> {4805 static TargetType to(SourceType src) {4806 KMP_ASSERT(src <= static_cast<SourceType>(4807 (std::numeric_limits<TargetType>::max)()));4808 KMP_ASSERT(src >= static_cast<SourceType>(4809 (std::numeric_limits<TargetType>::min)()));4810 return (TargetType)src;4811 }4812};4813 4814// Source signed, Target unsigned4815// Source smaller4816template <typename SourceType, typename TargetType>4817struct kmp_convert<SourceType, TargetType, true, false, true, false> {4818 static TargetType to(SourceType src) {4819 KMP_ASSERT(src >= 0);4820 return (TargetType)src;4821 }4822};4823// Source equal4824template <typename SourceType, typename TargetType>4825struct kmp_convert<SourceType, TargetType, false, true, true, false> {4826 static TargetType to(SourceType src) {4827 KMP_ASSERT(src >= 0);4828 return (TargetType)src;4829 }4830};4831// Source bigger4832template <typename SourceType, typename TargetType>4833struct kmp_convert<SourceType, TargetType, false, false, true, false> {4834 static TargetType to(SourceType src) {4835 KMP_ASSERT(src >= 0);4836 KMP_ASSERT(src <= static_cast<SourceType>(4837 (std::numeric_limits<TargetType>::max)()));4838 return (TargetType)src;4839 }4840};4841 4842// Source unsigned, Target signed4843// Source smaller4844template <typename SourceType, typename TargetType>4845struct kmp_convert<SourceType, TargetType, true, false, false, true> {4846 static TargetType to(SourceType src) { return (TargetType)src; }4847};4848// Source equal4849template <typename SourceType, typename TargetType>4850struct kmp_convert<SourceType, TargetType, false, true, false, true> {4851 static TargetType to(SourceType src) {4852 KMP_ASSERT(src <= static_cast<SourceType>(4853 (std::numeric_limits<TargetType>::max)()));4854 return (TargetType)src;4855 }4856};4857// Source bigger4858template <typename SourceType, typename TargetType>4859struct kmp_convert<SourceType, TargetType, false, false, false, true> {4860 static TargetType to(SourceType src) {4861 KMP_ASSERT(src <= static_cast<SourceType>(4862 (std::numeric_limits<TargetType>::max)()));4863 return (TargetType)src;4864 }4865};4866 4867// Source unsigned, Target unsigned4868// Source smaller4869template <typename SourceType, typename TargetType>4870struct kmp_convert<SourceType, TargetType, true, false, false, false> {4871 static TargetType to(SourceType src) { return (TargetType)src; }4872};4873// Source equal4874template <typename SourceType, typename TargetType>4875struct kmp_convert<SourceType, TargetType, false, true, false, false> {4876 static TargetType to(SourceType src) { return src; }4877};4878// Source bigger4879template <typename SourceType, typename TargetType>4880struct kmp_convert<SourceType, TargetType, false, false, false, false> {4881 static TargetType to(SourceType src) {4882 KMP_ASSERT(src <= static_cast<SourceType>(4883 (std::numeric_limits<TargetType>::max)()));4884 return (TargetType)src;4885 }4886};4887 4888template <typename T1, typename T2>4889static inline void __kmp_type_convert(T1 src, T2 *dest) {4890 *dest = kmp_convert<T1, T2>::to(src);4891}4892 4893#endif /* KMP_H */4894