571 lines · cpp
1//===-------- interface.cpp - Target independent OpenMP target RTL --------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Implementation of the interface to be used by Clang during the codegen of a10// target region.11//12//===----------------------------------------------------------------------===//13 14#include "OpenMP/OMPT/Interface.h"15#include "OffloadPolicy.h"16#include "OpenMP/OMPT/Callback.h"17#include "OpenMP/omp.h"18#include "PluginManager.h"19#include "omptarget.h"20#include "private.h"21 22#include "Shared/EnvironmentVar.h"23#include "Shared/Profile.h"24 25#include "Utils/ExponentialBackoff.h"26 27#include "llvm/Frontend/OpenMP/OMPConstants.h"28 29#include <cassert>30#include <cstdint>31#include <cstdio>32#include <cstdlib>33#include <memory>34 35#ifdef OMPT_SUPPORT36using namespace llvm::omp::target::ompt;37#endif38 39// If offload is enabled, ensure that device DeviceID has been initialized.40//41// The return bool indicates if the offload is to the host device42// There are three possible results:43// - Return false if the target device is ready for offload44// - Return true without reporting a runtime error if offload is45// disabled, perhaps because the initial device was specified.46// - Report a runtime error and return true.47//48// If DeviceID == OFFLOAD_DEVICE_DEFAULT, set DeviceID to the default device.49// This step might be skipped if offload is disabled.50bool checkDevice(int64_t &DeviceID, ident_t *Loc) {51 if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED) {52 DP("Offload is disabled\n");53 return true;54 }55 56 if (DeviceID == OFFLOAD_DEVICE_DEFAULT) {57 DeviceID = omp_get_default_device();58 DP("Use default device id %" PRId64 "\n", DeviceID);59 }60 61 // Proposed behavior for OpenMP 5.2 in OpenMP spec github issue 2669.62 if (omp_get_num_devices() == 0) {63 DP("omp_get_num_devices() == 0 but offload is manadatory\n");64 handleTargetOutcome(false, Loc);65 return true;66 }67 68 if (DeviceID == omp_get_initial_device()) {69 DP("Device is host (%" PRId64 "), returning as if offload is disabled\n",70 DeviceID);71 return true;72 }73 return false;74}75 76////////////////////////////////////////////////////////////////////////////////77/// adds requires flags78EXTERN void __tgt_register_requires(int64_t Flags) {79 MESSAGE("The %s function has been removed. Old OpenMP requirements will not "80 "be handled",81 __PRETTY_FUNCTION__);82}83 84EXTERN void __tgt_rtl_init() { initRuntime(); }85EXTERN void __tgt_rtl_deinit() { deinitRuntime(); }86 87////////////////////////////////////////////////////////////////////////////////88/// adds a target shared library to the target execution image89EXTERN void __tgt_register_lib(__tgt_bin_desc *Desc) {90 initRuntime();91 if (PM->delayRegisterLib(Desc))92 return;93 94 PM->registerLib(Desc);95}96 97////////////////////////////////////////////////////////////////////////////////98/// Initialize all available devices without registering any image99EXTERN void __tgt_init_all_rtls() {100 assert(PM && "Runtime not initialized");101 PM->initializeAllDevices();102}103 104////////////////////////////////////////////////////////////////////////////////105/// unloads a target shared library106EXTERN void __tgt_unregister_lib(__tgt_bin_desc *Desc) {107 PM->unregisterLib(Desc);108 109 deinitRuntime();110}111 112template <typename TargetAsyncInfoTy>113static inline void114targetData(ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,115 void **Args, int64_t *ArgSizes, int64_t *ArgTypes,116 map_var_info_t *ArgNames, void **ArgMappers,117 TargetDataFuncPtrTy TargetDataFunction, const char *RegionTypeMsg,118 const char *RegionName) {119 assert(PM && "Runtime not initialized");120 static_assert(std::is_convertible_v<TargetAsyncInfoTy &, AsyncInfoTy &>,121 "TargetAsyncInfoTy must be convertible to AsyncInfoTy.");122 123 TIMESCOPE_WITH_DETAILS_AND_IDENT("Runtime: Data Copy",124 "NumArgs=" + std::to_string(ArgNum), Loc);125 126 DP("Entering data %s region for device %" PRId64 " with %d mappings\n",127 RegionName, DeviceId, ArgNum);128 129 if (checkDevice(DeviceId, Loc)) {130 DP("Not offloading to device %" PRId64 "\n", DeviceId);131 return;132 }133 134 if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS)135 printKernelArguments(Loc, DeviceId, ArgNum, ArgSizes, ArgTypes, ArgNames,136 RegionTypeMsg);137#ifdef OMPTARGET_DEBUG138 for (int I = 0; I < ArgNum; ++I) {139 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64140 ", Type=0x%" PRIx64 ", Name=%s\n",141 I, DPxPTR(ArgsBase[I]), DPxPTR(Args[I]), ArgSizes[I], ArgTypes[I],142 (ArgNames) ? getNameFromMapping(ArgNames[I]).c_str() : "unknown");143 }144#endif145 146 auto DeviceOrErr = PM->getDevice(DeviceId);147 if (!DeviceOrErr)148 FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());149 150 TargetAsyncInfoTy TargetAsyncInfo(*DeviceOrErr);151 AsyncInfoTy &AsyncInfo = TargetAsyncInfo;152 153 /// RAII to establish tool anchors before and after data begin / end / update154 OMPT_IF_BUILT(assert((TargetDataFunction == targetDataBegin ||155 TargetDataFunction == targetDataEnd ||156 TargetDataFunction == targetDataUpdate) &&157 "Encountered unexpected TargetDataFunction during "158 "execution of targetData");159 auto CallbackFunctions =160 (TargetDataFunction == targetDataBegin)161 ? RegionInterface.getCallbacks<ompt_target_enter_data>()162 : (TargetDataFunction == targetDataEnd)163 ? RegionInterface.getCallbacks<ompt_target_exit_data>()164 : RegionInterface.getCallbacks<ompt_target_update>();165 InterfaceRAII TargetDataRAII(CallbackFunctions, DeviceId,166 OMPT_GET_RETURN_ADDRESS);)167 168 int Rc = OFFLOAD_SUCCESS;169 170 // Only allocate AttachInfo for targetDataBegin171 std::unique_ptr<AttachInfoTy> AttachInfo;172 if (TargetDataFunction == targetDataBegin)173 AttachInfo = std::make_unique<AttachInfoTy>();174 175 Rc = TargetDataFunction(Loc, *DeviceOrErr, ArgNum, ArgsBase, Args, ArgSizes,176 ArgTypes, ArgNames, ArgMappers, AsyncInfo,177 AttachInfo.get(), /*FromMapper=*/false);178 179 if (Rc == OFFLOAD_SUCCESS) {180 // Process deferred ATTACH entries BEFORE synchronization181 if (AttachInfo && !AttachInfo->AttachEntries.empty())182 Rc = processAttachEntries(*DeviceOrErr, *AttachInfo, AsyncInfo);183 184 if (Rc == OFFLOAD_SUCCESS)185 Rc = AsyncInfo.synchronize();186 }187 188 handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc);189}190 191/// creates host-to-target data mapping, stores it in the192/// libomptarget.so internal structure (an entry in a stack of data maps)193/// and passes the data to the device.194EXTERN void __tgt_target_data_begin_mapper(ident_t *Loc, int64_t DeviceId,195 int32_t ArgNum, void **ArgsBase,196 void **Args, int64_t *ArgSizes,197 int64_t *ArgTypes,198 map_var_info_t *ArgNames,199 void **ArgMappers) {200 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));201 targetData<AsyncInfoTy>(Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes,202 ArgTypes, ArgNames, ArgMappers, targetDataBegin,203 "Entering OpenMP data region with being_mapper",204 "begin");205}206 207EXTERN void __tgt_target_data_begin_nowait_mapper(208 ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,209 void **Args, int64_t *ArgSizes, int64_t *ArgTypes, map_var_info_t *ArgNames,210 void **ArgMappers, int32_t DepNum, void *DepList, int32_t NoAliasDepNum,211 void *NoAliasDepList) {212 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));213 targetData<TaskAsyncInfoWrapperTy>(214 Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames,215 ArgMappers, targetDataBegin,216 "Entering OpenMP data region with being_nowait_mapper", "begin");217}218 219/// passes data from the target, releases target memory and destroys220/// the host-target mapping (top entry from the stack of data maps)221/// created by the last __tgt_target_data_begin.222EXTERN void __tgt_target_data_end_mapper(ident_t *Loc, int64_t DeviceId,223 int32_t ArgNum, void **ArgsBase,224 void **Args, int64_t *ArgSizes,225 int64_t *ArgTypes,226 map_var_info_t *ArgNames,227 void **ArgMappers) {228 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));229 targetData<AsyncInfoTy>(Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes,230 ArgTypes, ArgNames, ArgMappers, targetDataEnd,231 "Exiting OpenMP data region with end_mapper", "end");232}233 234EXTERN void __tgt_target_data_end_nowait_mapper(235 ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,236 void **Args, int64_t *ArgSizes, int64_t *ArgTypes, map_var_info_t *ArgNames,237 void **ArgMappers, int32_t DepNum, void *DepList, int32_t NoAliasDepNum,238 void *NoAliasDepList) {239 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));240 targetData<TaskAsyncInfoWrapperTy>(241 Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames,242 ArgMappers, targetDataEnd,243 "Exiting OpenMP data region with end_nowait_mapper", "end");244}245 246EXTERN void __tgt_target_data_update_mapper(ident_t *Loc, int64_t DeviceId,247 int32_t ArgNum, void **ArgsBase,248 void **Args, int64_t *ArgSizes,249 int64_t *ArgTypes,250 map_var_info_t *ArgNames,251 void **ArgMappers) {252 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));253 targetData<AsyncInfoTy>(254 Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames,255 ArgMappers, targetDataUpdate,256 "Updating data within the OpenMP data region with update_mapper",257 "update");258}259 260EXTERN void __tgt_target_data_update_nowait_mapper(261 ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,262 void **Args, int64_t *ArgSizes, int64_t *ArgTypes, map_var_info_t *ArgNames,263 void **ArgMappers, int32_t DepNum, void *DepList, int32_t NoAliasDepNum,264 void *NoAliasDepList) {265 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));266 targetData<TaskAsyncInfoWrapperTy>(267 Loc, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames,268 ArgMappers, targetDataUpdate,269 "Updating data within the OpenMP data region with update_nowait_mapper",270 "update");271}272 273static KernelArgsTy *upgradeKernelArgs(KernelArgsTy *KernelArgs,274 KernelArgsTy &LocalKernelArgs,275 int32_t NumTeams, int32_t ThreadLimit) {276 if (KernelArgs->Version > OMP_KERNEL_ARG_VERSION)277 DP("Unexpected ABI version: %u\n", KernelArgs->Version);278 279 uint32_t UpgradedVersion = KernelArgs->Version;280 if (KernelArgs->Version < OMP_KERNEL_ARG_VERSION) {281 // The upgraded version will be based on the kernel launch environment.282 if (KernelArgs->Version < OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR)283 UpgradedVersion = OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR - 1;284 else285 UpgradedVersion = OMP_KERNEL_ARG_VERSION;286 }287 if (UpgradedVersion != KernelArgs->Version) {288 LocalKernelArgs.Version = UpgradedVersion;289 LocalKernelArgs.NumArgs = KernelArgs->NumArgs;290 LocalKernelArgs.ArgBasePtrs = KernelArgs->ArgBasePtrs;291 LocalKernelArgs.ArgPtrs = KernelArgs->ArgPtrs;292 LocalKernelArgs.ArgSizes = KernelArgs->ArgSizes;293 LocalKernelArgs.ArgTypes = KernelArgs->ArgTypes;294 LocalKernelArgs.ArgNames = KernelArgs->ArgNames;295 LocalKernelArgs.ArgMappers = KernelArgs->ArgMappers;296 LocalKernelArgs.Tripcount = KernelArgs->Tripcount;297 LocalKernelArgs.Flags = KernelArgs->Flags;298 LocalKernelArgs.DynCGroupMem = 0;299 LocalKernelArgs.NumTeams[0] = NumTeams;300 LocalKernelArgs.NumTeams[1] = 1;301 LocalKernelArgs.NumTeams[2] = 1;302 LocalKernelArgs.ThreadLimit[0] = ThreadLimit;303 LocalKernelArgs.ThreadLimit[1] = 1;304 LocalKernelArgs.ThreadLimit[2] = 1;305 return &LocalKernelArgs;306 }307 308 // FIXME: This is a WA to "calibrate" the bad work done in the front end.309 // Delete this ugly code after the front end emits proper values.310 auto CorrectMultiDim = [](uint32_t (&Val)[3]) {311 if (Val[1] == 0)312 Val[1] = 1;313 if (Val[2] == 0)314 Val[2] = 1;315 };316 CorrectMultiDim(KernelArgs->ThreadLimit);317 CorrectMultiDim(KernelArgs->NumTeams);318 319 return KernelArgs;320}321 322template <typename TargetAsyncInfoTy>323static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,324 int32_t ThreadLimit, void *HostPtr,325 KernelArgsTy *KernelArgs) {326 assert(PM && "Runtime not initialized");327 static_assert(std::is_convertible_v<TargetAsyncInfoTy &, AsyncInfoTy &>,328 "Target AsyncInfoTy must be convertible to AsyncInfoTy.");329 DP("Entering target region for device %" PRId64 " with entry point " DPxMOD330 "\n",331 DeviceId, DPxPTR(HostPtr));332 333 if (checkDevice(DeviceId, Loc)) {334 DP("Not offloading to device %" PRId64 "\n", DeviceId);335 return OMP_TGT_FAIL;336 }337 338 bool IsTeams = NumTeams != -1;339 if (!IsTeams)340 KernelArgs->NumTeams[0] = NumTeams = 1;341 342 // Auto-upgrade kernel args version 1 to 2.343 KernelArgsTy LocalKernelArgs;344 KernelArgs =345 upgradeKernelArgs(KernelArgs, LocalKernelArgs, NumTeams, ThreadLimit);346 347 TIMESCOPE_WITH_DETAILS_AND_IDENT(348 "Runtime: target exe",349 "NumTeams=" + std::to_string(NumTeams) +350 ";NumArgs=" + std::to_string(KernelArgs->NumArgs),351 Loc);352 353 if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS)354 printKernelArguments(Loc, DeviceId, KernelArgs->NumArgs,355 KernelArgs->ArgSizes, KernelArgs->ArgTypes,356 KernelArgs->ArgNames, "Entering OpenMP kernel");357#ifdef OMPTARGET_DEBUG358 for (uint32_t I = 0; I < KernelArgs->NumArgs; ++I) {359 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64360 ", Type=0x%" PRIx64 ", Name=%s\n",361 I, DPxPTR(KernelArgs->ArgBasePtrs[I]), DPxPTR(KernelArgs->ArgPtrs[I]),362 KernelArgs->ArgSizes[I], KernelArgs->ArgTypes[I],363 (KernelArgs->ArgNames)364 ? getNameFromMapping(KernelArgs->ArgNames[I]).c_str()365 : "unknown");366 }367#endif368 369 auto DeviceOrErr = PM->getDevice(DeviceId);370 if (!DeviceOrErr)371 FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());372 373 TargetAsyncInfoTy TargetAsyncInfo(*DeviceOrErr);374 AsyncInfoTy &AsyncInfo = TargetAsyncInfo;375 /// RAII to establish tool anchors before and after target region376 OMPT_IF_BUILT(InterfaceRAII TargetRAII(377 RegionInterface.getCallbacks<ompt_target>(), DeviceId,378 /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)379 380 int Rc = OFFLOAD_SUCCESS;381 Rc = target(Loc, *DeviceOrErr, HostPtr, *KernelArgs, AsyncInfo);382 { // required to show synchronization383 TIMESCOPE_WITH_DETAILS_AND_IDENT("Runtime: synchronize", "", Loc);384 if (Rc == OFFLOAD_SUCCESS)385 Rc = AsyncInfo.synchronize();386 387 handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc);388 assert(Rc == OFFLOAD_SUCCESS && "__tgt_target_kernel unexpected failure!");389 }390 return OMP_TGT_SUCCESS;391}392 393/// Implements a kernel entry that executes the target region on the specified394/// device.395///396/// \param Loc Source location associated with this target region.397/// \param DeviceId The device to execute this region, -1 indicated the default.398/// \param NumTeams Number of teams to launch the region with, -1 indicates a399/// non-teams region and 0 indicates it was unspecified.400/// \param ThreadLimit Limit to the number of threads to use in the kernel401/// launch, 0 indicates it was unspecified.402/// \param HostPtr The pointer to the host function registered with the kernel.403/// \param Args All arguments to this kernel launch (see struct definition).404EXTERN int __tgt_target_kernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,405 int32_t ThreadLimit, void *HostPtr,406 KernelArgsTy *KernelArgs) {407 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));408 if (KernelArgs->Flags.NoWait)409 return targetKernel<TaskAsyncInfoWrapperTy>(410 Loc, DeviceId, NumTeams, ThreadLimit, HostPtr, KernelArgs);411 return targetKernel<AsyncInfoTy>(Loc, DeviceId, NumTeams, ThreadLimit,412 HostPtr, KernelArgs);413}414 415/// Activates the record replay mechanism.416/// \param DeviceId The device identifier to execute the target region.417/// \param MemorySize The number of bytes to be (pre-)allocated418/// by the bump allocator419/// /param IsRecord Activates the record replay mechanism in420/// 'record' mode or 'replay' mode.421/// /param SaveOutput Store the device memory after kernel422/// execution on persistent storage423EXTERN int __tgt_activate_record_replay(int64_t DeviceId, uint64_t MemorySize,424 void *VAddr, bool IsRecord,425 bool SaveOutput,426 uint64_t &ReqPtrArgOffset) {427 assert(PM && "Runtime not initialized");428 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));429 auto DeviceOrErr = PM->getDevice(DeviceId);430 if (!DeviceOrErr)431 FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());432 433 [[maybe_unused]] int Rc = target_activate_rr(434 *DeviceOrErr, MemorySize, VAddr, IsRecord, SaveOutput, ReqPtrArgOffset);435 assert(Rc == OFFLOAD_SUCCESS &&436 "__tgt_activate_record_replay unexpected failure!");437 return OMP_TGT_SUCCESS;438}439 440/// Implements a target kernel entry that replays a pre-recorded kernel.441/// \param Loc Source location associated with this target region (unused).442/// \param DeviceId The device identifier to execute the target region.443/// \param HostPtr A pointer to an address that uniquely identifies the kernel.444/// \param DeviceMemory A pointer to an array storing device memory data to move445/// prior to kernel execution.446/// \param DeviceMemorySize The size of the above device memory data in bytes.447/// \param TgtArgs An array of pointers of the pre-recorded target kernel448/// arguments.449/// \param TgtOffsets An array of pointers of the pre-recorded target kernel450/// argument offsets.451/// \param NumArgs The number of kernel arguments.452/// \param NumTeams Number of teams to launch the target region with.453/// \param ThreadLimit Limit to the number of threads to use in kernel454/// execution.455/// \param LoopTripCount The pre-recorded value of the loop tripcount, if any.456/// \return OMP_TGT_SUCCESS on success, OMP_TGT_FAIL on failure.457EXTERN int __tgt_target_kernel_replay(ident_t *Loc, int64_t DeviceId,458 void *HostPtr, void *DeviceMemory,459 int64_t DeviceMemorySize, void **TgtArgs,460 ptrdiff_t *TgtOffsets, int32_t NumArgs,461 int32_t NumTeams, int32_t ThreadLimit,462 uint64_t LoopTripCount) {463 assert(PM && "Runtime not initialized");464 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));465 if (checkDevice(DeviceId, Loc)) {466 DP("Not offloading to device %" PRId64 "\n", DeviceId);467 return OMP_TGT_FAIL;468 }469 auto DeviceOrErr = PM->getDevice(DeviceId);470 if (!DeviceOrErr)471 FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());472 473 /// RAII to establish tool anchors before and after target region474 OMPT_IF_BUILT(InterfaceRAII TargetRAII(475 RegionInterface.getCallbacks<ompt_target>(), DeviceId,476 /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)477 478 AsyncInfoTy AsyncInfo(*DeviceOrErr);479 int Rc = target_replay(Loc, *DeviceOrErr, HostPtr, DeviceMemory,480 DeviceMemorySize, TgtArgs, TgtOffsets, NumArgs,481 NumTeams, ThreadLimit, LoopTripCount, AsyncInfo);482 if (Rc == OFFLOAD_SUCCESS)483 Rc = AsyncInfo.synchronize();484 handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc);485 assert(Rc == OFFLOAD_SUCCESS &&486 "__tgt_target_kernel_replay unexpected failure!");487 return OMP_TGT_SUCCESS;488}489 490// Get the current number of components for a user-defined mapper.491EXTERN int64_t __tgt_mapper_num_components(void *RtMapperHandle) {492 auto *MapperComponentsPtr = (struct MapperComponentsTy *)RtMapperHandle;493 int64_t Size = MapperComponentsPtr->Components.size();494 DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n",495 DPxPTR(RtMapperHandle), Size);496 return Size;497}498 499// Push back one component for a user-defined mapper.500EXTERN void __tgt_push_mapper_component(void *RtMapperHandle, void *Base,501 void *Begin, int64_t Size, int64_t Type,502 void *Name) {503 DP("__tgt_push_mapper_component(Handle=" DPxMOD504 ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64505 ", Type=0x%" PRIx64 ", Name=%s).\n",506 DPxPTR(RtMapperHandle), DPxPTR(Base), DPxPTR(Begin), Size, Type,507 (Name) ? getNameFromMapping(Name).c_str() : "unknown");508 auto *MapperComponentsPtr = (struct MapperComponentsTy *)RtMapperHandle;509 MapperComponentsPtr->Components.push_back(510 MapComponentInfoTy(Base, Begin, Size, Type, Name));511}512 513EXTERN void __tgt_set_info_flag(uint32_t NewInfoLevel) {514 assert(PM && "Runtime not initialized");515 std::atomic<uint32_t> &InfoLevel = getInfoLevelInternal();516 InfoLevel.store(NewInfoLevel);517}518 519EXTERN int __tgt_print_device_info(int64_t DeviceId) {520 assert(PM && "Runtime not initialized");521 auto DeviceOrErr = PM->getDevice(DeviceId);522 if (!DeviceOrErr)523 FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());524 525 return DeviceOrErr->printDeviceInfo();526}527 528EXTERN void __tgt_target_nowait_query(void **AsyncHandle) {529 assert(PM && "Runtime not initialized");530 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));531 532 if (!AsyncHandle || !*AsyncHandle) {533 FATAL_MESSAGE0(534 1, "Receive an invalid async handle from the current OpenMP task. Is "535 "this a target nowait region?\n");536 }537 538 // Exponential backoff tries to optimally decide if a thread should just query539 // for the device operations (work/spin wait on them) or block until they are540 // completed (use device side blocking mechanism). This allows the runtime to541 // adapt itself when there are a lot of long-running target regions in-flight.542 static thread_local utils::ExponentialBackoff QueryCounter(543 Int64Envar("OMPTARGET_QUERY_COUNT_MAX", 10),544 Int64Envar("OMPTARGET_QUERY_COUNT_THRESHOLD", 5),545 Envar<float>("OMPTARGET_QUERY_COUNT_BACKOFF_FACTOR", 0.5f));546 547 auto *AsyncInfo = (AsyncInfoTy *)*AsyncHandle;548 549 // If the thread is actively waiting on too many target nowait regions, we550 // should use the blocking sync type.551 if (QueryCounter.isAboveThreshold())552 AsyncInfo->SyncType = AsyncInfoTy::SyncTy::BLOCKING;553 554 if (AsyncInfo->synchronize())555 FATAL_MESSAGE0(1, "Error while querying the async queue for completion.\n");556 // If there are device operations still pending, return immediately without557 // deallocating the handle and increase the current thread query count.558 if (!AsyncInfo->isDone()) {559 QueryCounter.increment();560 return;561 }562 563 // When a thread successfully completes a target nowait region, we564 // exponentially backoff its query counter by the query factor.565 QueryCounter.decrement();566 567 // Delete the handle and unset it from the OpenMP task data.568 delete AsyncInfo;569 *AsyncHandle = nullptr;570}571