469 lines · cpp
1//===------ State.cpp - OpenMP State & ICV interface ------------- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9//===----------------------------------------------------------------------===//10 11#include "Shared/Environment.h"12 13#include "Allocator.h"14#include "Configuration.h"15#include "Debug.h"16#include "DeviceTypes.h"17#include "DeviceUtils.h"18#include "Interface.h"19#include "LibC.h"20#include "Mapping.h"21#include "State.h"22#include "Synchronization.h"23 24using namespace ompx;25 26/// Memory implementation27///28///{29 30/// External symbol to access dynamic shared memory.31[[gnu::aligned(32 allocator::ALIGNMENT)]] extern Local<unsigned char> DynamicSharedBuffer[];33 34/// The kernel environment passed to the init method by the compiler.35[[clang::loader_uninitialized]] static Local<KernelEnvironmentTy *>36 KernelEnvironmentPtr;37 38/// The kernel launch environment passed as argument to the kernel by the39/// runtime.40[[clang::loader_uninitialized]] static Local<KernelLaunchEnvironmentTy *>41 KernelLaunchEnvironmentPtr;42 43///}44 45namespace {46 47/// A "smart" stack in shared memory.48///49/// The stack exposes a malloc/free interface but works like a stack internally.50/// In fact, it is a separate stack *per warp*. That means, each warp must push51/// and pop symmetrically or this breaks, badly. The implementation will (aim52/// to) detect non-lock-step warps and fallback to malloc/free. The same will53/// happen if a warp runs out of memory. The master warp in generic memory is54/// special and is given more memory than the rest.55///56struct SharedMemorySmartStackTy {57 /// Initialize the stack. Must be called by all threads.58 void init(bool IsSPMD);59 60 /// Allocate \p Bytes on the stack for the encountering thread. Each thread61 /// can call this function.62 void *push(uint64_t Bytes);63 64 /// Deallocate the last allocation made by the encountering thread and pointed65 /// to by \p Ptr from the stack. Each thread can call this function.66 void pop(void *Ptr, uint64_t Bytes);67 68private:69 /// Compute the size of the storage space reserved for a thread.70 uint32_t computeThreadStorageTotal() {71 uint32_t NumLanesInBlock = mapping::getNumberOfThreadsInBlock();72 return __builtin_align_down(state::SharedScratchpadSize / NumLanesInBlock,73 allocator::ALIGNMENT);74 }75 76 /// Return the top address of the warp data stack, that is the first address77 /// this warp will allocate memory at next.78 void *getThreadDataTop(uint32_t TId) {79 return &Data[computeThreadStorageTotal() * TId + Usage[TId]];80 }81 82 /// The actual storage, shared among all warps.83 [[gnu::aligned(84 allocator::ALIGNMENT)]] unsigned char Data[state::SharedScratchpadSize];85 [[gnu::aligned(86 allocator::ALIGNMENT)]] unsigned char Usage[mapping::MaxThreadsPerTeam];87};88 89static_assert(state::SharedScratchpadSize / mapping::MaxThreadsPerTeam <= 256,90 "Shared scratchpad of this size not supported yet.");91 92/// The allocation of a single shared memory scratchpad.93[[clang::loader_uninitialized]] static Local<SharedMemorySmartStackTy>94 SharedMemorySmartStack;95 96void SharedMemorySmartStackTy::init(bool IsSPMD) {97 Usage[mapping::getThreadIdInBlock()] = 0;98}99 100void *SharedMemorySmartStackTy::push(uint64_t Bytes) {101 // First align the number of requested bytes.102 /// FIXME: The stack shouldn't require worst-case padding. Alignment needs to103 /// be passed in as an argument and the stack rewritten to support it.104 uint64_t AlignedBytes = __builtin_align_up(Bytes, allocator::ALIGNMENT);105 106 uint32_t StorageTotal = computeThreadStorageTotal();107 108 // The main thread in generic mode gets the space of its entire warp as the109 // other threads do not participate in any computation at all.110 if (mapping::isMainThreadInGenericMode())111 StorageTotal *= mapping::getWarpSize();112 113 int TId = mapping::getThreadIdInBlock();114 if (Usage[TId] + AlignedBytes <= StorageTotal) {115 void *Ptr = getThreadDataTop(TId);116 Usage[TId] += AlignedBytes;117 return Ptr;118 }119 120 if (config::isDebugMode(DeviceDebugKind::CommonIssues))121 printf("Shared memory stack full, fallback to dynamic allocation of global "122 "memory will negatively impact performance.\n");123 void *GlobalMemory = memory::allocGlobal(124 AlignedBytes, "Slow path shared memory allocation, insufficient "125 "shared memory stack memory!");126 ASSERT(GlobalMemory != nullptr, "nullptr returned by malloc!");127 128 return GlobalMemory;129}130 131void SharedMemorySmartStackTy::pop(void *Ptr, uint64_t Bytes) {132 uint64_t AlignedBytes = __builtin_align_up(Bytes, allocator::ALIGNMENT);133 if (utils::isSharedMemPtr(Ptr)) {134 int TId = mapping::getThreadIdInBlock();135 Usage[TId] -= AlignedBytes;136 return;137 }138 memory::freeGlobal(Ptr, "Slow path shared memory deallocation");139}140 141} // namespace142 143void *memory::getDynamicBuffer() { return DynamicSharedBuffer; }144 145void *memory::allocShared(uint64_t Bytes, const char *Reason) {146 return SharedMemorySmartStack.push(Bytes);147}148 149void memory::freeShared(void *Ptr, uint64_t Bytes, const char *Reason) {150 SharedMemorySmartStack.pop(Ptr, Bytes);151}152 153void *memory::allocGlobal(uint64_t Bytes, const char *Reason) {154 void *Ptr = allocator::alloc(Bytes);155 if (config::isDebugMode(DeviceDebugKind::CommonIssues) && Ptr == nullptr)156 printf("nullptr returned by malloc!\n");157 return Ptr;158}159 160void memory::freeGlobal(void *Ptr, const char *Reason) { allocator::free(Ptr); }161 162///}163 164bool state::ICVStateTy::operator==(const ICVStateTy &Other) const {165 return (NThreadsVar == Other.NThreadsVar) & (LevelVar == Other.LevelVar) &166 (ActiveLevelVar == Other.ActiveLevelVar) &167 (MaxActiveLevelsVar == Other.MaxActiveLevelsVar) &168 (RunSchedVar == Other.RunSchedVar) &169 (RunSchedChunkVar == Other.RunSchedChunkVar);170}171 172void state::ICVStateTy::assertEqual(const ICVStateTy &Other) const {173 ASSERT(NThreadsVar == Other.NThreadsVar, nullptr);174 ASSERT(LevelVar == Other.LevelVar, nullptr);175 ASSERT(ActiveLevelVar == Other.ActiveLevelVar, nullptr);176 ASSERT(MaxActiveLevelsVar == Other.MaxActiveLevelsVar, nullptr);177 ASSERT(RunSchedVar == Other.RunSchedVar, nullptr);178 ASSERT(RunSchedChunkVar == Other.RunSchedChunkVar, nullptr);179}180 181void state::TeamStateTy::init(bool IsSPMD) {182 ICVState.NThreadsVar = 0;183 ICVState.LevelVar = 0;184 ICVState.ActiveLevelVar = 0;185 ICVState.Padding0Val = 0;186 ICVState.MaxActiveLevelsVar = 1;187 ICVState.RunSchedVar = omp_sched_static;188 ICVState.RunSchedChunkVar = 1;189 ParallelTeamSize = 1;190 HasThreadState = false;191 ParallelRegionFnVar = nullptr;192}193 194bool state::TeamStateTy::operator==(const TeamStateTy &Other) const {195 return (ICVState == Other.ICVState) &196 (HasThreadState == Other.HasThreadState) &197 (ParallelTeamSize == Other.ParallelTeamSize);198}199 200void state::TeamStateTy::assertEqual(TeamStateTy &Other) const {201 ICVState.assertEqual(Other.ICVState);202 ASSERT(ParallelTeamSize == Other.ParallelTeamSize, nullptr);203 ASSERT(HasThreadState == Other.HasThreadState, nullptr);204}205 206[[clang::loader_uninitialized]] Local<state::TeamStateTy>207 ompx::state::TeamState;208[[clang::loader_uninitialized]] Local<state::ThreadStateTy **>209 ompx::state::ThreadStates;210 211namespace {212 213int returnValIfLevelIsActive(int Level, int Val, int DefaultVal,214 int OutOfBoundsVal = -1) {215 if (Level == 0)216 return DefaultVal;217 int LevelVar = omp_get_level();218 if (OMP_UNLIKELY(Level < 0 || Level > LevelVar))219 return OutOfBoundsVal;220 int ActiveLevel = icv::ActiveLevel;221 if (OMP_UNLIKELY(Level != ActiveLevel))222 return DefaultVal;223 return Val;224}225 226} // namespace227 228void state::init(bool IsSPMD, KernelEnvironmentTy &KernelEnvironment,229 KernelLaunchEnvironmentTy &KernelLaunchEnvironment) {230 SharedMemorySmartStack.init(IsSPMD);231 if (mapping::isInitialThreadInLevel0(IsSPMD)) {232 TeamState.init(IsSPMD);233 ThreadStates = nullptr;234 KernelEnvironmentPtr = &KernelEnvironment;235 KernelLaunchEnvironmentPtr = &KernelLaunchEnvironment;236 }237}238 239KernelEnvironmentTy &state::getKernelEnvironment() {240 return *KernelEnvironmentPtr;241}242 243KernelLaunchEnvironmentTy &state::getKernelLaunchEnvironment() {244 return *KernelLaunchEnvironmentPtr;245}246 247void state::enterDataEnvironment(IdentTy *Ident) {248 ASSERT(config::mayUseThreadStates(),249 "Thread state modified while explicitly disabled!");250 if (!config::mayUseThreadStates())251 return;252 253 unsigned TId = mapping::getThreadIdInBlock();254 ThreadStateTy *NewThreadState = static_cast<ThreadStateTy *>(255 memory::allocGlobal(sizeof(ThreadStateTy), "ThreadStates alloc"));256 uintptr_t *ThreadStatesBitsPtr = reinterpret_cast<uintptr_t *>(&ThreadStates);257 if (!atomic::load(ThreadStatesBitsPtr, atomic::seq_cst)) {258 uint32_t Bytes =259 sizeof(ThreadStates[0]) * mapping::getNumberOfThreadsInBlock();260 void *ThreadStatesPtr =261 memory::allocGlobal(Bytes, "Thread state array allocation");262 __builtin_memset(ThreadStatesPtr, 0, Bytes);263 if (!atomic::cas(ThreadStatesBitsPtr, uintptr_t(0),264 reinterpret_cast<uintptr_t>(ThreadStatesPtr),265 atomic::seq_cst, atomic::seq_cst))266 memory::freeGlobal(ThreadStatesPtr,267 "Thread state array allocated multiple times");268 ASSERT(atomic::load(ThreadStatesBitsPtr, atomic::seq_cst),269 "Expected valid thread states bit!");270 }271 NewThreadState->init(ThreadStates[TId]);272 TeamState.HasThreadState = true;273 ThreadStates[TId] = NewThreadState;274}275 276void state::exitDataEnvironment() {277 ASSERT(config::mayUseThreadStates(),278 "Thread state modified while explicitly disabled!");279 280 unsigned TId = mapping::getThreadIdInBlock();281 resetStateForThread(TId);282}283 284void state::resetStateForThread(uint32_t TId) {285 if (!config::mayUseThreadStates())286 return;287 if (OMP_LIKELY(!TeamState.HasThreadState || !ThreadStates[TId]))288 return;289 290 ThreadStateTy *PreviousThreadState = ThreadStates[TId]->PreviousThreadState;291 memory::freeGlobal(ThreadStates[TId], "ThreadStates dealloc");292 ThreadStates[TId] = PreviousThreadState;293}294 295void state::runAndCheckState(void(Func(void))) {296 TeamStateTy OldTeamState = TeamState;297 OldTeamState.assertEqual(TeamState);298 299 Func();300 301 OldTeamState.assertEqual(TeamState);302}303 304void state::assumeInitialState(bool IsSPMD) {305 TeamStateTy InitialTeamState;306 InitialTeamState.init(IsSPMD);307 InitialTeamState.assertEqual(TeamState);308 ASSERT(mapping::isSPMDMode() == IsSPMD, nullptr);309}310 311int state::getEffectivePTeamSize() {312 int PTeamSize = state::ParallelTeamSize;313 return PTeamSize ? PTeamSize : mapping::getMaxTeamThreads();314}315 316extern "C" {317void omp_set_dynamic(int V) {}318 319int omp_get_dynamic(void) { return 0; }320 321void omp_set_num_threads(int V) { icv::NThreads = V; }322 323int omp_get_max_threads(void) {324 int NT = icv::NThreads;325 return NT > 0 ? NT : mapping::getMaxTeamThreads();326}327 328int omp_get_level(void) {329 int LevelVar = icv::Level;330 ASSERT(LevelVar >= 0, nullptr);331 return LevelVar;332}333 334int omp_get_active_level(void) { return !!icv::ActiveLevel; }335 336int omp_in_parallel(void) { return !!icv::ActiveLevel; }337 338void omp_get_schedule(omp_sched_t *ScheduleKind, int *ChunkSize) {339 *ScheduleKind = static_cast<omp_sched_t>((int)icv::RunSched);340 *ChunkSize = state::RunSchedChunk;341}342 343void omp_set_schedule(omp_sched_t ScheduleKind, int ChunkSize) {344 icv::RunSched = (int)ScheduleKind;345 state::RunSchedChunk = ChunkSize;346}347 348int omp_get_ancestor_thread_num(int Level) {349 return returnValIfLevelIsActive(Level, mapping::getThreadIdInBlock(), 0);350}351 352int omp_get_thread_num(void) {353 return omp_get_ancestor_thread_num(omp_get_level());354}355 356int omp_get_team_size(int Level) {357 return returnValIfLevelIsActive(Level, state::getEffectivePTeamSize(), 1);358}359 360int omp_get_num_threads(void) {361 return omp_get_level() != 1 ? 1 : state::getEffectivePTeamSize();362}363 364int omp_get_thread_limit(void) { return mapping::getMaxTeamThreads(); }365 366int omp_get_num_procs(void) { return mapping::getNumberOfProcessorElements(); }367 368void omp_set_nested(int) {}369 370int omp_get_nested(void) { return false; }371 372void omp_set_max_active_levels(int Levels) {373 icv::MaxActiveLevels = Levels > 0 ? 1 : 0;374}375 376int omp_get_max_active_levels(void) { return icv::MaxActiveLevels; }377 378omp_proc_bind_t omp_get_proc_bind(void) { return omp_proc_bind_false; }379 380int omp_get_num_places(void) { return 0; }381 382int omp_get_place_num_procs(int) { return omp_get_num_procs(); }383 384void omp_get_place_proc_ids(int, int *) {385 // TODO386}387 388int omp_get_place_num(void) { return 0; }389 390int omp_get_partition_num_places(void) { return 0; }391 392void omp_get_partition_place_nums(int *) {393 // TODO394}395 396int omp_get_cancellation(void) { return 0; }397 398void omp_set_default_device(int) {}399 400int omp_get_default_device(void) { return -1; }401 402int omp_get_num_devices(void) { return config::getNumDevices(); }403 404int omp_get_device_num(void) { return config::getDeviceNum(); }405 406int omp_get_device_from_uid(const char *DeviceUid) {407 return omp_invalid_device;408}409 410const char *omp_get_uid_from_device(int DeviceNum) { return nullptr; }411 412int omp_get_num_teams(void) { return mapping::getNumberOfBlocksInKernel(); }413 414int omp_get_team_num() { return mapping::getBlockIdInKernel(); }415 416int omp_get_initial_device(void) { return -1; }417 418int omp_is_initial_device(void) { return 0; }419}420 421extern "C" {422[[clang::noinline]] void *__kmpc_alloc_shared(uint64_t Bytes) {423 return memory::allocShared(Bytes, "Frontend alloc shared");424}425 426[[clang::noinline]] void __kmpc_free_shared(void *Ptr, uint64_t Bytes) {427 memory::freeShared(Ptr, Bytes, "Frontend free shared");428}429 430void *__kmpc_get_dynamic_shared() { return memory::getDynamicBuffer(); }431 432void *llvm_omp_target_dynamic_shared_alloc() {433 return __kmpc_get_dynamic_shared();434}435 436void *llvm_omp_get_dynamic_shared() { return __kmpc_get_dynamic_shared(); }437 438/// Allocate storage in shared memory to communicate arguments from the main439/// thread to the workers in generic mode. If we exceed440/// NUM_SHARED_VARIABLES_IN_SHARED_MEM we will malloc space for communication.441constexpr uint64_t NUM_SHARED_VARIABLES_IN_SHARED_MEM = 64;442 443[[clang::loader_uninitialized]] static Local<void *>444 SharedMemVariableSharingSpace[NUM_SHARED_VARIABLES_IN_SHARED_MEM];445[[clang::loader_uninitialized]] static Local<void **>446 SharedMemVariableSharingSpacePtr;447 448void __kmpc_begin_sharing_variables(void ***GlobalArgs, uint64_t nArgs) {449 if (nArgs <= NUM_SHARED_VARIABLES_IN_SHARED_MEM) {450 SharedMemVariableSharingSpacePtr = &SharedMemVariableSharingSpace[0];451 } else {452 SharedMemVariableSharingSpacePtr = (void **)memory::allocGlobal(453 nArgs * sizeof(void *), "new extended args");454 ASSERT(SharedMemVariableSharingSpacePtr != nullptr,455 "Nullptr returned by malloc!");456 }457 *GlobalArgs = SharedMemVariableSharingSpacePtr;458}459 460void __kmpc_end_sharing_variables() {461 if (SharedMemVariableSharingSpacePtr != &SharedMemVariableSharingSpace[0])462 memory::freeGlobal(SharedMemVariableSharingSpacePtr, "new extended args");463}464 465void __kmpc_get_shared_variables(void ***GlobalArgs) {466 *GlobalArgs = SharedMemVariableSharingSpacePtr;467}468}469