438 lines · c
1//===-------- omptarget.h - Target independent OpenMP target RTL -- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Interface to be used by Clang during the codegen of a10// target region.11//12//===----------------------------------------------------------------------===//13 14#ifndef _OMPTARGET_H_15#define _OMPTARGET_H_16 17#include "Shared/APITypes.h"18#include "Shared/Environment.h"19#include "Shared/SourceInfo.h"20 21#include "OpenMP/InternalTypes.h"22 23#include <cstddef>24#include <cstdint>25#include <deque>26#include <functional>27#include <type_traits>28 29#include "llvm/ADT/SmallVector.h"30 31#define OFFLOAD_SUCCESS (0)32#define OFFLOAD_FAIL (~0)33 34#define OFFLOAD_DEVICE_DEFAULT -135 36/// return flags of __tgt_target_XXX public APIs37enum __tgt_target_return_t : int {38 /// successful offload executed on a target device39 OMP_TGT_SUCCESS = 0,40 /// offload may not execute on the requested target device41 /// this scenario can be caused by the device not available or unsupported42 /// as described in the Execution Model in the specification43 /// this status may not be used for target device execution failure44 /// which should be handled internally in libomptarget45 OMP_TGT_FAIL = ~046};47 48/// Data attributes for each data reference used in an OpenMP target region.49enum tgt_map_type {50 // No flags51 OMP_TGT_MAPTYPE_NONE = 0x000,52 // copy data from host to device53 OMP_TGT_MAPTYPE_TO = 0x001,54 // copy data from device to host55 OMP_TGT_MAPTYPE_FROM = 0x002,56 // copy regardless of the reference count57 OMP_TGT_MAPTYPE_ALWAYS = 0x004,58 // force unmapping of data59 OMP_TGT_MAPTYPE_DELETE = 0x008,60 // map the pointer as well as the pointee61 OMP_TGT_MAPTYPE_PTR_AND_OBJ = 0x010,62 // pass device base address to kernel63 OMP_TGT_MAPTYPE_TARGET_PARAM = 0x020,64 // return base device address of mapped data65 OMP_TGT_MAPTYPE_RETURN_PARAM = 0x040,66 // private variable - not mapped67 OMP_TGT_MAPTYPE_PRIVATE = 0x080,68 // copy by value - not mapped69 OMP_TGT_MAPTYPE_LITERAL = 0x100,70 // mapping is implicit71 OMP_TGT_MAPTYPE_IMPLICIT = 0x200,72 // copy data to device73 OMP_TGT_MAPTYPE_CLOSE = 0x400,74 // runtime error if not already allocated75 OMP_TGT_MAPTYPE_PRESENT = 0x1000,76 // use a separate reference counter so that the data cannot be unmapped within77 // the structured region78 // This is an OpenMP extension for the sake of OpenACC support.79 OMP_TGT_MAPTYPE_OMPX_HOLD = 0x2000,80 // Attach pointer and pointee, after processing all other maps.81 // Applicable to map-entering directives. Does not change ref-count.82 OMP_TGT_MAPTYPE_ATTACH = 0x4000,83 // descriptor for non-contiguous target-update84 OMP_TGT_MAPTYPE_NON_CONTIG = 0x100000000000,85 // member of struct, member given by [16 MSBs] - 186 OMP_TGT_MAPTYPE_MEMBER_OF = 0xffff00000000000087};88 89/// Flags for offload entries.90enum OpenMPOffloadingDeclareTargetFlags {91 /// Mark the entry global as having a 'link' attribute.92 OMP_DECLARE_TARGET_LINK = 0x01,93 /// Mark the entry global as being an indirectly callable function.94 OMP_DECLARE_TARGET_INDIRECT = 0x08,95 /// This is an entry corresponding to a requirement to be registered.96 OMP_REGISTER_REQUIRES = 0x10,97 /// Mark the entry global as being an indirect vtable.98 OMP_DECLARE_TARGET_INDIRECT_VTABLE = 0x20,99};100 101enum TargetAllocTy : int32_t {102 TARGET_ALLOC_DEVICE = 0,103 TARGET_ALLOC_HOST,104 TARGET_ALLOC_SHARED,105 TARGET_ALLOC_DEFAULT,106};107 108struct DeviceTy;109 110/// The libomptarget wrapper around a __tgt_async_info object directly111/// associated with a libomptarget layer device. RAII semantics to avoid112/// mistakes.113class AsyncInfoTy {114public:115 enum class SyncTy { BLOCKING, NON_BLOCKING };116 117private:118 /// Locations we used in (potentially) asynchronous calls which should live119 /// as long as this AsyncInfoTy object.120 std::deque<void *> BufferLocations;121 122 /// Post-processing operations executed after a successful synchronization.123 /// \note the post-processing function should return OFFLOAD_SUCCESS or124 /// OFFLOAD_FAIL appropriately.125 using PostProcFuncTy = std::function<int()>;126 llvm::SmallVector<PostProcFuncTy> PostProcessingFunctions;127 128 __tgt_async_info AsyncInfo;129 DeviceTy &Device;130 131public:132 /// Synchronization method to be used.133 SyncTy SyncType;134 135 AsyncInfoTy(DeviceTy &Device, SyncTy SyncType = SyncTy::BLOCKING)136 : Device(Device), SyncType(SyncType) {}137 ~AsyncInfoTy() { synchronize(); }138 139 /// Implicit conversion to the __tgt_async_info which is used in the140 /// plugin interface.141 operator __tgt_async_info *() { return &AsyncInfo; }142 143 /// Synchronize all pending actions.144 ///145 /// \note synchronization will be performance in a blocking or non-blocking146 /// manner, depending on the SyncType.147 ///148 /// \note if the operations are completed, the registered post-processing149 /// functions will be executed once and unregistered afterwards.150 ///151 /// \returns OFFLOAD_FAIL or OFFLOAD_SUCCESS appropriately.152 int synchronize();153 154 /// Return a void* reference with a lifetime that is at least as long as this155 /// AsyncInfoTy object. The location can be used as intermediate buffer.156 void *&getVoidPtrLocation();157 158 /// Check if all asynchronous operations are completed.159 ///160 /// \note only a lightweight check. If needed, use synchronize() to query the161 /// status of AsyncInfo before checking.162 ///163 /// \returns true if there is no pending asynchronous operations, false164 /// otherwise.165 bool isDone() const;166 167 /// Add a new post-processing function to be executed after synchronization.168 ///169 /// \param[in] Function is a templated function (e.g., function pointers,170 /// lambdas, std::function) that can be convertible to a PostProcFuncTy (i.e.,171 /// it must have int() as its function signature).172 template <typename FuncTy> void addPostProcessingFunction(FuncTy &&Function) {173 static_assert(std::is_convertible_v<FuncTy, PostProcFuncTy>,174 "Invalid post-processing function type. Please check "175 "function signature!");176 PostProcessingFunctions.emplace_back(Function);177 }178 179private:180 /// Run all the post-processing functions sequentially.181 ///182 /// \note after a successful execution, all previously registered functions183 /// are unregistered.184 ///185 /// \returns OFFLOAD_FAIL if any post-processing function failed,186 /// OFFLOAD_SUCCESS otherwise.187 int32_t runPostProcessing();188 189 /// Check if the internal asynchronous info queue is empty or not.190 ///191 /// \returns true if empty, false otherwise.192 bool isQueueEmpty() const;193};194 195// Wrapper for task stored async info objects.196class TaskAsyncInfoWrapperTy {197 // Invalid GTID as defined by libomp; keep in sync198 static constexpr int KMP_GTID_DNE = -2;199 200 const int ExecThreadID = KMP_GTID_DNE;201 AsyncInfoTy LocalAsyncInfo;202 AsyncInfoTy *AsyncInfo = &LocalAsyncInfo;203 void **TaskAsyncInfoPtr = nullptr;204 205public:206 TaskAsyncInfoWrapperTy(DeviceTy &Device)207 : ExecThreadID(__kmpc_global_thread_num(NULL)), LocalAsyncInfo(Device) {208 // If we failed to acquired the current global thread id, we cannot209 // re-enqueue the current task. Thus we should use the local blocking async210 // info.211 if (ExecThreadID == KMP_GTID_DNE)212 return;213 214 // Only tasks with an assigned task team can be re-enqueue and thus can215 // use the non-blocking synchronization scheme. Thus we should use the local216 // blocking async info, if we don´t have one.217 if (!__kmpc_omp_has_task_team(ExecThreadID))218 return;219 220 // Acquire a pointer to the AsyncInfo stored inside the current task being221 // executed.222 TaskAsyncInfoPtr = __kmpc_omp_get_target_async_handle_ptr(ExecThreadID);223 224 // If we cannot acquire such pointer, fallback to using the local blocking225 // async info.226 if (!TaskAsyncInfoPtr)227 return;228 229 // When creating a new task async info, the task handle must always be230 // invalid. We must never overwrite any task async handle and there should231 // never be any valid handle store inside the task at this point.232 assert((*TaskAsyncInfoPtr) == nullptr &&233 "Task async handle is not empty when dispatching new device "234 "operations. The handle was not cleared properly or "235 "__tgt_target_nowait_query should have been called!");236 237 // If no valid async handle is present, a new AsyncInfo will be allocated238 // and stored in the current task.239 AsyncInfo = new AsyncInfoTy(Device, AsyncInfoTy::SyncTy::NON_BLOCKING);240 *TaskAsyncInfoPtr = (void *)AsyncInfo;241 }242 243 ~TaskAsyncInfoWrapperTy() {244 // Local async info destruction is automatically handled by ~AsyncInfoTy.245 if (AsyncInfo == &LocalAsyncInfo)246 return;247 248 // If the are device operations still pending, return immediately without249 // deallocating the handle.250 if (!AsyncInfo->isDone())251 return;252 253 // Delete the handle and unset it from the OpenMP task data.254 delete AsyncInfo;255 *TaskAsyncInfoPtr = nullptr;256 }257 258 operator AsyncInfoTy &() { return *AsyncInfo; }259};260 261/// This struct is a record of non-contiguous information262struct __tgt_target_non_contig {263 uint64_t Offset;264 uint64_t Count;265 uint64_t Stride;266};267 268#ifdef __cplusplus269extern "C" {270#endif271 272void ompx_dump_mapping_tables(void);273int omp_get_num_devices(void);274int omp_get_device_num(void);275int omp_get_device_from_uid(const char *DeviceUid);276const char *omp_get_uid_from_device(int DeviceNum);277int omp_get_initial_device(void);278void *omp_target_alloc(size_t Size, int DeviceNum);279void omp_target_free(void *DevicePtr, int DeviceNum);280int omp_target_is_present(const void *Ptr, int DeviceNum);281int omp_target_is_accessible(const void *Ptr, size_t Size, int DeviceNum);282int omp_target_memcpy(void *Dst, const void *Src, size_t Length,283 size_t DstOffset, size_t SrcOffset, int DstDevice,284 int SrcDevice);285int omp_target_memcpy_rect(void *Dst, const void *Src, size_t ElementSize,286 int NumDims, const size_t *Volume,287 const size_t *DstOffsets, const size_t *SrcOffsets,288 const size_t *DstDimensions,289 const size_t *SrcDimensions, int DstDevice,290 int SrcDevice);291void *omp_target_memset(void *Ptr, int C, size_t N, int DeviceNum);292int omp_target_associate_ptr(const void *HostPtr, const void *DevicePtr,293 size_t Size, size_t DeviceOffset, int DeviceNum);294int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum);295 296/// Explicit target memory allocators297/// Using the llvm_ prefix until they become part of the OpenMP standard.298void *llvm_omp_target_alloc_device(size_t Size, int DeviceNum);299void *llvm_omp_target_alloc_host(size_t Size, int DeviceNum);300void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum);301 302/// Explicit target memory deallocators303/// Using the llvm_ prefix until they become part of the OpenMP standard.304void llvm_omp_target_free_device(void *DevicePtr, int DeviceNum);305void llvm_omp_target_free_host(void *DevicePtr, int DeviceNum);306void llvm_omp_target_free_shared(void *DevicePtr, int DeviceNum);307 308/// Dummy target so we have a symbol for generating host fallback.309void *llvm_omp_target_dynamic_shared_alloc();310 311/// add the clauses of the requires directives in a given file312void __tgt_register_requires(int64_t Flags);313 314/// Initializes the runtime library.315void __tgt_rtl_init();316 317/// Deinitializes the runtime library.318void __tgt_rtl_deinit();319 320/// adds a target shared library to the target execution image321void __tgt_register_lib(__tgt_bin_desc *Desc);322 323/// Initialize all RTLs at once324void __tgt_init_all_rtls();325 326/// removes a target shared library from the target execution image327void __tgt_unregister_lib(__tgt_bin_desc *Desc);328 329// creates the host to target data mapping, stores it in the330// libomptarget.so internal structure (an entry in a stack of data maps) and331// passes the data to the device;332void __tgt_target_data_begin(int64_t DeviceId, int32_t ArgNum, void **ArgsBase,333 void **Args, int64_t *ArgSizes, int64_t *ArgTypes);334void __tgt_target_data_begin_nowait(int64_t DeviceId, int32_t ArgNum,335 void **ArgsBase, void **Args,336 int64_t *ArgSizes, int64_t *ArgTypes,337 int32_t DepNum, void *DepList,338 int32_t NoAliasDepNum,339 void *NoAliasDepList);340void __tgt_target_data_begin_mapper(ident_t *Loc, int64_t DeviceId,341 int32_t ArgNum, void **ArgsBase,342 void **Args, int64_t *ArgSizes,343 int64_t *ArgTypes, map_var_info_t *ArgNames,344 void **ArgMappers);345void __tgt_target_data_begin_nowait_mapper(346 ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,347 void **Args, int64_t *ArgSizes, int64_t *ArgTypes, map_var_info_t *ArgNames,348 void **ArgMappers, int32_t DepNum, void *DepList, int32_t NoAliasDepNum,349 void *NoAliasDepList);350 351// passes data from the target, release target memory and destroys the352// host-target mapping (top entry from the stack of data maps) created by353// the last __tgt_target_data_begin354void __tgt_target_data_end(int64_t DeviceId, int32_t ArgNum, void **ArgsBase,355 void **Args, int64_t *ArgSizes, int64_t *ArgTypes);356void __tgt_target_data_end_nowait(int64_t DeviceId, int32_t ArgNum,357 void **ArgsBase, void **Args,358 int64_t *ArgSizes, int64_t *ArgTypes,359 int32_t DepNum, void *DepList,360 int32_t NoAliasDepNum, void *NoAliasDepList);361void __tgt_target_data_end_mapper(ident_t *Loc, int64_t DeviceId,362 int32_t ArgNum, void **ArgsBase, void **Args,363 int64_t *ArgSizes, int64_t *ArgTypes,364 map_var_info_t *ArgNames, void **ArgMappers);365void __tgt_target_data_end_nowait_mapper(366 ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,367 void **Args, int64_t *ArgSizes, int64_t *ArgTypes, map_var_info_t *ArgNames,368 void **ArgMappers, int32_t depNum, void *depList, int32_t NoAliasDepNum,369 void *NoAliasDepList);370 371/// passes data to/from the target372void __tgt_target_data_update(int64_t DeviceId, int32_t ArgNum, void **ArgsBase,373 void **Args, int64_t *ArgSizes,374 int64_t *ArgTypes);375void __tgt_target_data_update_nowait(int64_t DeviceId, int32_t ArgNum,376 void **ArgsBase, void **Args,377 int64_t *ArgSizes, int64_t *ArgTypes,378 int32_t DepNum, void *DepList,379 int32_t NoAliasDepNum,380 void *NoAliasDepList);381void __tgt_target_data_update_mapper(ident_t *Loc, int64_t DeviceId,382 int32_t ArgNum, void **ArgsBase,383 void **Args, int64_t *ArgSizes,384 int64_t *ArgTypes,385 map_var_info_t *ArgNames,386 void **ArgMappers);387void __tgt_target_data_update_nowait_mapper(388 ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,389 void **Args, int64_t *ArgSizes, int64_t *ArgTypes, map_var_info_t *ArgNames,390 void **ArgMappers, int32_t DepNum, void *DepList, int32_t NoAliasDepNum,391 void *NoAliasDepList);392 393// Performs the same actions as data_begin in case ArgNum is non-zero394// and initiates run of offloaded region on target platform; if ArgNum395// is non-zero after the region execution is done it also performs the396// same action as data_end above. The following types are used; this397// function returns 0 if it was able to transfer the execution to a398// target and an int different from zero otherwise.399int __tgt_target_kernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,400 int32_t ThreadLimit, void *HostPtr, KernelArgsTy *Args);401 402// Non-blocking synchronization for target nowait regions. This function403// acquires the asynchronous context from task data of the current task being404// executed and tries to query for the completion of its operations. If the405// operations are still pending, the function returns immediately. If the406// operations are completed, all the post-processing procedures stored in the407// asynchronous context are executed and the context is removed from the task408// data.409void __tgt_target_nowait_query(void **AsyncHandle);410 411/// Executes a target kernel by replaying recorded kernel arguments and412/// device memory.413int __tgt_target_kernel_replay(ident_t *Loc, int64_t DeviceId, void *HostPtr,414 void *DeviceMemory, int64_t DeviceMemorySize,415 void **TgtArgs, ptrdiff_t *TgtOffsets,416 int32_t NumArgs, int32_t NumTeams,417 int32_t ThreadLimit, uint64_t LoopTripCount);418 419void __tgt_set_info_flag(uint32_t);420 421int __tgt_print_device_info(int64_t DeviceId);422 423int __tgt_activate_record_replay(int64_t DeviceId, uint64_t MemorySize,424 void *VAddr, bool IsRecord, bool SaveOutput,425 uint64_t &ReqPtrArgOffset);426 427#ifdef __cplusplus428}429#endif430 431#ifdef __cplusplus432#define EXTERN extern "C"433#else434#define EXTERN extern435#endif436 437#endif // _OMPTARGET_H_438