768 lines · cpp
1//===----------- api.cpp - Target independent OpenMP target RTL -----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Implementation of OpenMP API interface functions.10//11//===----------------------------------------------------------------------===//12 13#include "PluginManager.h"14#include "device.h"15#include "omptarget.h"16#include "rtl.h"17 18#include "OpenMP/InternalTypes.h"19#include "OpenMP/InteropAPI.h"20#include "OpenMP/Mapping.h"21#include "OpenMP/OMPT/Interface.h"22#include "OpenMP/omp.h"23#include "Shared/Profile.h"24 25#include "llvm/ADT/SmallVector.h"26 27#include <climits>28#include <cstdlib>29#include <cstring>30#include <mutex>31 32EXTERN void ompx_dump_mapping_tables() {33 ident_t Loc = {0, 0, 0, 0, ";libomptarget;libomptarget;0;0;;"};34 auto ExclusiveDevicesAccessor = PM->getExclusiveDevicesAccessor();35 for (auto &Device : PM->devices(ExclusiveDevicesAccessor))36 dumpTargetPointerMappings(&Loc, Device, true);37}38 39#ifdef OMPT_SUPPORT40using namespace llvm::omp::target::ompt;41#endif42 43using GenericDeviceTy = llvm::omp::target::plugin::GenericDeviceTy;44 45void *targetAllocExplicit(size_t Size, int DeviceNum, int Kind,46 const char *Name);47void targetFreeExplicit(void *DevicePtr, int DeviceNum, int Kind,48 const char *Name);49void *targetLockExplicit(void *HostPtr, size_t Size, int DeviceNum,50 const char *Name);51void targetUnlockExplicit(void *HostPtr, int DeviceNum, const char *Name);52 53EXTERN int omp_get_num_devices(void) {54 TIMESCOPE();55 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));56 size_t NumDevices = PM->getNumDevices();57 58 DP("Call to omp_get_num_devices returning %zd\n", NumDevices);59 60 return NumDevices;61}62 63EXTERN int omp_get_device_num(void) {64 TIMESCOPE();65 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));66 int HostDevice = omp_get_initial_device();67 68 DP("Call to omp_get_device_num returning %d\n", HostDevice);69 70 return HostDevice;71}72 73static inline bool is_initial_device_uid(const char *DeviceUid) {74 return strcmp(DeviceUid, GenericPluginTy::getHostDeviceUid()) == 0;75}76 77EXTERN int omp_get_device_from_uid(const char *DeviceUid) {78 TIMESCOPE();79 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));80 81 if (!DeviceUid) {82 DP("Call to omp_get_device_from_uid returning omp_invalid_device\n");83 return omp_invalid_device;84 }85 if (is_initial_device_uid(DeviceUid)) {86 DP("Call to omp_get_device_from_uid returning initial device number %d\n",87 omp_get_initial_device());88 return omp_get_initial_device();89 }90 91 int DeviceNum = omp_invalid_device;92 93 auto ExclusiveDevicesAccessor = PM->getExclusiveDevicesAccessor();94 for (const DeviceTy &Device : PM->devices(ExclusiveDevicesAccessor)) {95 const char *Uid = Device.RTL->getDevice(Device.RTLDeviceID).getDeviceUid();96 if (Uid && strcmp(DeviceUid, Uid) == 0) {97 DeviceNum = Device.DeviceID;98 break;99 }100 }101 102 DP("Call to omp_get_device_from_uid returning %d\n", DeviceNum);103 return DeviceNum;104}105 106EXTERN const char *omp_get_uid_from_device(int DeviceNum) {107 TIMESCOPE();108 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));109 110 if (DeviceNum == omp_invalid_device) {111 DP("Call to omp_get_uid_from_device returning nullptr\n");112 return nullptr;113 }114 if (DeviceNum == omp_get_initial_device()) {115 DP("Call to omp_get_uid_from_device returning initial device UID\n");116 return GenericPluginTy::getHostDeviceUid();117 }118 119 auto DeviceOrErr = PM->getDevice(DeviceNum);120 if (!DeviceOrErr)121 FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());122 123 const char *Uid =124 DeviceOrErr->RTL->getDevice(DeviceOrErr->RTLDeviceID).getDeviceUid();125 DP("Call to omp_get_uid_from_device returning %s\n", Uid);126 return Uid;127}128 129EXTERN int omp_get_initial_device(void) {130 TIMESCOPE();131 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));132 int HostDevice = omp_get_num_devices();133 DP("Call to omp_get_initial_device returning %d\n", HostDevice);134 return HostDevice;135}136 137EXTERN void *omp_target_alloc(size_t Size, int DeviceNum) {138 TIMESCOPE_WITH_DETAILS("dst_dev=" + std::to_string(DeviceNum) +139 ";size=" + std::to_string(Size));140 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));141 return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_DEFAULT, __func__);142}143 144EXTERN void *llvm_omp_target_alloc_device(size_t Size, int DeviceNum) {145 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));146 return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_DEVICE, __func__);147}148 149EXTERN void *llvm_omp_target_alloc_host(size_t Size, int DeviceNum) {150 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));151 return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_HOST, __func__);152}153 154EXTERN void *llvm_omp_target_alloc_shared(size_t Size, int DeviceNum) {155 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));156 return targetAllocExplicit(Size, DeviceNum, TARGET_ALLOC_SHARED, __func__);157}158 159EXTERN void omp_target_free(void *Ptr, int DeviceNum) {160 TIMESCOPE();161 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));162 return targetFreeExplicit(Ptr, DeviceNum, TARGET_ALLOC_DEFAULT, __func__);163}164 165EXTERN void llvm_omp_target_free_device(void *Ptr, int DeviceNum) {166 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));167 return targetFreeExplicit(Ptr, DeviceNum, TARGET_ALLOC_DEVICE, __func__);168}169 170EXTERN void llvm_omp_target_free_host(void *Ptr, int DeviceNum) {171 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));172 return targetFreeExplicit(Ptr, DeviceNum, TARGET_ALLOC_HOST, __func__);173}174 175EXTERN void llvm_omp_target_free_shared(void *Ptre, int DeviceNum) {176 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));177 return targetFreeExplicit(Ptre, DeviceNum, TARGET_ALLOC_SHARED, __func__);178}179 180EXTERN void *llvm_omp_target_dynamic_shared_alloc() {181 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));182 return nullptr;183}184 185EXTERN void *llvm_omp_get_dynamic_shared() {186 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));187 return nullptr;188}189 190EXTERN [[nodiscard]] void *llvm_omp_target_lock_mem(void *Ptr, size_t Size,191 int DeviceNum) {192 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));193 return targetLockExplicit(Ptr, Size, DeviceNum, __func__);194}195 196EXTERN void llvm_omp_target_unlock_mem(void *Ptr, int DeviceNum) {197 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));198 targetUnlockExplicit(Ptr, DeviceNum, __func__);199}200 201EXTERN int omp_target_is_present(const void *Ptr, int DeviceNum) {202 TIMESCOPE();203 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));204 DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",205 DeviceNum, DPxPTR(Ptr));206 207 if (!Ptr) {208 DP("Call to omp_target_is_present with NULL ptr, returning false\n");209 return false;210 }211 212 if (DeviceNum == omp_get_initial_device()) {213 DP("Call to omp_target_is_present on host, returning true\n");214 return true;215 }216 217 auto DeviceOrErr = PM->getDevice(DeviceNum);218 if (!DeviceOrErr)219 FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());220 221 // omp_target_is_present tests whether a host pointer refers to storage that222 // is mapped to a given device. However, due to the lack of the storage size,223 // only check 1 byte. Cannot set size 0 which checks whether the pointer (zero224 // length array) is mapped instead of the referred storage.225 TargetPointerResultTy TPR =226 DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,227 /*UpdateRefCount=*/false,228 /*UseHoldRefCount=*/false);229 int Rc = TPR.isPresent();230 DP("Call to omp_target_is_present returns %d\n", Rc);231 return Rc;232}233 234/// Check whether a pointer is accessible from a device.235/// Returns true when accessibility is guaranteed otherwise returns false.236EXTERN int omp_target_is_accessible(const void *Ptr, size_t Size,237 int DeviceNum) {238 TIMESCOPE();239 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));240 DP("Call to omp_target_is_accessible for device %d, address " DPxMOD241 ", size %zu\n",242 DeviceNum, DPxPTR(Ptr), Size);243 244 if (!Ptr) {245 DP("Call to omp_target_is_accessible with NULL ptr returning false\n");246 return false;247 }248 249 if (DeviceNum == omp_get_initial_device() || DeviceNum == -1) {250 DP("Call to omp_target_is_accessible on host, returning true\n");251 return true;252 }253 254 // The device number must refer to a valid device255 auto DeviceOrErr = PM->getDevice(DeviceNum);256 if (!DeviceOrErr)257 FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());258 259 return DeviceOrErr->isAccessiblePtr(Ptr, Size);260}261 262EXTERN int omp_target_memcpy(void *Dst, const void *Src, size_t Length,263 size_t DstOffset, size_t SrcOffset, int DstDevice,264 int SrcDevice) {265 TIMESCOPE_WITH_DETAILS("dst_dev=" + std::to_string(DstDevice) +266 ";src_dev=" + std::to_string(SrcDevice) +267 ";size=" + std::to_string(Length));268 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));269 DP("Call to omp_target_memcpy, dst device %d, src device %d, "270 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "271 "src offset %zu, length %zu\n",272 DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DstOffset, SrcOffset,273 Length);274 275 if (!Dst || !Src || Length <= 0) {276 if (Length == 0) {277 DP("Call to omp_target_memcpy with zero length, nothing to do\n");278 return OFFLOAD_SUCCESS;279 }280 281 REPORT("Call to omp_target_memcpy with invalid arguments\n");282 return OFFLOAD_FAIL;283 }284 285 int Rc = OFFLOAD_SUCCESS;286 void *SrcAddr = (char *)const_cast<void *>(Src) + SrcOffset;287 void *DstAddr = (char *)Dst + DstOffset;288 289 if (SrcDevice == omp_get_initial_device() &&290 DstDevice == omp_get_initial_device()) {291 DP("copy from host to host\n");292 const void *P = memcpy(DstAddr, SrcAddr, Length);293 if (P == NULL)294 Rc = OFFLOAD_FAIL;295 } else if (SrcDevice == omp_get_initial_device()) {296 DP("copy from host to device\n");297 auto DstDeviceOrErr = PM->getDevice(DstDevice);298 if (!DstDeviceOrErr)299 FATAL_MESSAGE(DstDevice, "%s",300 toString(DstDeviceOrErr.takeError()).c_str());301 AsyncInfoTy AsyncInfo(*DstDeviceOrErr);302 Rc = DstDeviceOrErr->submitData(DstAddr, SrcAddr, Length, AsyncInfo);303 } else if (DstDevice == omp_get_initial_device()) {304 DP("copy from device to host\n");305 auto SrcDeviceOrErr = PM->getDevice(SrcDevice);306 if (!SrcDeviceOrErr)307 FATAL_MESSAGE(SrcDevice, "%s",308 toString(SrcDeviceOrErr.takeError()).c_str());309 AsyncInfoTy AsyncInfo(*SrcDeviceOrErr);310 Rc = SrcDeviceOrErr->retrieveData(DstAddr, SrcAddr, Length, AsyncInfo);311 } else {312 DP("copy from device to device\n");313 auto SrcDeviceOrErr = PM->getDevice(SrcDevice);314 if (!SrcDeviceOrErr)315 FATAL_MESSAGE(SrcDevice, "%s",316 toString(SrcDeviceOrErr.takeError()).c_str());317 AsyncInfoTy AsyncInfo(*SrcDeviceOrErr);318 auto DstDeviceOrErr = PM->getDevice(DstDevice);319 if (!DstDeviceOrErr)320 FATAL_MESSAGE(DstDevice, "%s",321 toString(DstDeviceOrErr.takeError()).c_str());322 // First try to use D2D memcpy which is more efficient. If fails, fall back323 // to inefficient way.324 if (SrcDeviceOrErr->isDataExchangable(*DstDeviceOrErr)) {325 AsyncInfoTy AsyncInfo(*SrcDeviceOrErr);326 Rc = SrcDeviceOrErr->dataExchange(SrcAddr, *DstDeviceOrErr, DstAddr,327 Length, AsyncInfo);328 if (Rc == OFFLOAD_SUCCESS)329 return OFFLOAD_SUCCESS;330 }331 332 void *Buffer = malloc(Length);333 {334 AsyncInfoTy AsyncInfo(*SrcDeviceOrErr);335 Rc = SrcDeviceOrErr->retrieveData(Buffer, SrcAddr, Length, AsyncInfo);336 }337 if (Rc == OFFLOAD_SUCCESS) {338 AsyncInfoTy AsyncInfo(*DstDeviceOrErr);339 Rc = DstDeviceOrErr->submitData(DstAddr, Buffer, Length, AsyncInfo);340 }341 free(Buffer);342 }343 344 DP("omp_target_memcpy returns %d\n", Rc);345 return Rc;346}347 348// The helper function that calls omp_target_memcpy or omp_target_memcpy_rect349static int libomp_target_memcpy_async_task(int32_t Gtid, kmp_task_t *Task) {350 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));351 if (Task == nullptr)352 return OFFLOAD_FAIL;353 354 TargetMemcpyArgsTy *Args = (TargetMemcpyArgsTy *)Task->shareds;355 356 if (Args == nullptr)357 return OFFLOAD_FAIL;358 359 // Call blocked version360 int Rc = OFFLOAD_SUCCESS;361 if (Args->IsRectMemcpy) {362 Rc = omp_target_memcpy_rect(363 Args->Dst, Args->Src, Args->ElementSize, Args->NumDims, Args->Volume,364 Args->DstOffsets, Args->SrcOffsets, Args->DstDimensions,365 Args->SrcDimensions, Args->DstDevice, Args->SrcDevice);366 367 DP("omp_target_memcpy_rect returns %d\n", Rc);368 } else {369 Rc = omp_target_memcpy(Args->Dst, Args->Src, Args->Length, Args->DstOffset,370 Args->SrcOffset, Args->DstDevice, Args->SrcDevice);371 372 DP("omp_target_memcpy returns %d\n", Rc);373 }374 375 // Release the arguments object376 delete Args;377 378 return Rc;379}380 381static int libomp_target_memset_async_task(int32_t Gtid, kmp_task_t *Task) {382 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));383 if (!Task)384 return OFFLOAD_FAIL;385 386 auto *Args = reinterpret_cast<TargetMemsetArgsTy *>(Task->shareds);387 if (!Args)388 return OFFLOAD_FAIL;389 390 // call omp_target_memset()391 omp_target_memset(Args->Ptr, Args->C, Args->N, Args->DeviceNum);392 393 delete Args;394 395 return OFFLOAD_SUCCESS;396}397 398static inline void399convertDepObjVector(llvm::SmallVector<kmp_depend_info_t> &Vec, int DepObjCount,400 omp_depend_t *DepObjList) {401 for (int i = 0; i < DepObjCount; ++i) {402 omp_depend_t DepObj = DepObjList[i];403 Vec.push_back(*((kmp_depend_info_t *)DepObj));404 }405}406 407template <class T>408static inline int409libomp_helper_task_creation(T *Args, int (*Fn)(int32_t, kmp_task_t *),410 int DepObjCount, omp_depend_t *DepObjList) {411 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));412 // Create global thread ID413 int Gtid = __kmpc_global_thread_num(nullptr);414 415 // Setup the hidden helper flags416 int32_t Flags = 0;417 kmp_tasking_flags_t *InputFlags = (kmp_tasking_flags_t *)&Flags;418 InputFlags->hidden_helper = 1;419 420 // Alloc the helper task421 kmp_task_t *Task = __kmpc_omp_target_task_alloc(422 nullptr, Gtid, Flags, sizeof(kmp_task_t), 0, Fn, -1);423 if (!Task) {424 delete Args;425 return OFFLOAD_FAIL;426 }427 428 // Setup the arguments for the helper task429 Task->shareds = Args;430 431 // Convert types of depend objects432 llvm::SmallVector<kmp_depend_info_t> DepObjs;433 convertDepObjVector(DepObjs, DepObjCount, DepObjList);434 435 // Launch the helper task436 int Rc = __kmpc_omp_task_with_deps(nullptr, Gtid, Task, DepObjCount,437 DepObjs.data(), 0, nullptr);438 439 return Rc;440}441 442EXTERN void *omp_target_memset(void *Ptr, int ByteVal, size_t NumBytes,443 int DeviceNum) {444 TIMESCOPE();445 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));446 DP("Call to omp_target_memset, device %d, device pointer %p, size %zu\n",447 DeviceNum, Ptr, NumBytes);448 449 // Behave as a no-op if N==0 or if Ptr is nullptr (as a useful implementation450 // of unspecified behavior, see OpenMP spec).451 if (!Ptr || NumBytes == 0) {452 return Ptr;453 }454 455 if (DeviceNum == omp_get_initial_device()) {456 DP("filling memory on host via memset");457 memset(Ptr, ByteVal, NumBytes); // ignore return value, memset() cannot fail458 } else {459 // TODO: replace the omp_target_memset() slow path with the fast path.460 // That will require the ability to execute a kernel from within461 // libomptarget.so (which we do not have at the moment).462 463 // This is a very slow path: create a filled array on the host and upload464 // it to the GPU device.465 int InitialDevice = omp_get_initial_device();466 void *Shadow = omp_target_alloc(NumBytes, InitialDevice);467 if (Shadow) {468 (void)memset(Shadow, ByteVal, NumBytes);469 (void)omp_target_memcpy(Ptr, Shadow, NumBytes, 0, 0, DeviceNum,470 InitialDevice);471 (void)omp_target_free(Shadow, InitialDevice);472 } else {473 // If the omp_target_alloc has failed, let's just not do anything.474 // omp_target_memset does not have any good way to fail, so we475 // simply avoid a catastrophic failure of the process for now.476 DP("omp_target_memset failed to fill memory due to error with "477 "omp_target_alloc");478 }479 }480 481 DP("omp_target_memset returns %p\n", Ptr);482 return Ptr;483}484 485EXTERN void *omp_target_memset_async(void *Ptr, int ByteVal, size_t NumBytes,486 int DeviceNum, int DepObjCount,487 omp_depend_t *DepObjList) {488 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));489 DP("Call to omp_target_memset_async, device %d, device pointer %p, size %zu",490 DeviceNum, Ptr, NumBytes);491 492 // Behave as a no-op if N==0 or if Ptr is nullptr (as a useful implementation493 // of unspecified behavior, see OpenMP spec).494 if (!Ptr || NumBytes == 0)495 return Ptr;496 497 // Create the task object to deal with the async invocation498 auto *Args = new TargetMemsetArgsTy{Ptr, ByteVal, NumBytes, DeviceNum};499 500 // omp_target_memset_async() cannot fail via a return code, so ignore the501 // return code of the helper function502 (void)libomp_helper_task_creation(Args, &libomp_target_memset_async_task,503 DepObjCount, DepObjList);504 505 return Ptr;506}507 508EXTERN int omp_target_memcpy_async(void *Dst, const void *Src, size_t Length,509 size_t DstOffset, size_t SrcOffset,510 int DstDevice, int SrcDevice,511 int DepObjCount, omp_depend_t *DepObjList) {512 TIMESCOPE_WITH_DETAILS("dst_dev=" + std::to_string(DstDevice) +513 ";src_dev=" + std::to_string(SrcDevice) +514 ";size=" + std::to_string(Length));515 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));516 DP("Call to omp_target_memcpy_async, dst device %d, src device %d, "517 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "518 "src offset %zu, length %zu\n",519 DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DstOffset, SrcOffset,520 Length);521 522 // Check the source and dest address523 if (Dst == nullptr || Src == nullptr)524 return OFFLOAD_FAIL;525 526 // Create task object527 TargetMemcpyArgsTy *Args = new TargetMemcpyArgsTy(528 Dst, Src, Length, DstOffset, SrcOffset, DstDevice, SrcDevice);529 530 // Create and launch helper task531 int Rc = libomp_helper_task_creation(Args, &libomp_target_memcpy_async_task,532 DepObjCount, DepObjList);533 534 DP("omp_target_memcpy_async returns %d\n", Rc);535 return Rc;536}537 538EXTERN int539omp_target_memcpy_rect(void *Dst, const void *Src, size_t ElementSize,540 int NumDims, const size_t *Volume,541 const size_t *DstOffsets, const size_t *SrcOffsets,542 const size_t *DstDimensions, const size_t *SrcDimensions,543 int DstDevice, int SrcDevice) {544 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));545 DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "546 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "547 "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "548 "volume " DPxMOD ", element size %zu, num_dims %d\n",549 DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DPxPTR(DstOffsets),550 DPxPTR(SrcOffsets), DPxPTR(DstDimensions), DPxPTR(SrcDimensions),551 DPxPTR(Volume), ElementSize, NumDims);552 553 if (!(Dst || Src)) {554 DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",555 INT_MAX);556 return INT_MAX;557 }558 559 if (!Dst || !Src || ElementSize < 1 || NumDims < 1 || !Volume ||560 !DstOffsets || !SrcOffsets || !DstDimensions || !SrcDimensions) {561 REPORT("Call to omp_target_memcpy_rect with invalid arguments\n");562 return OFFLOAD_FAIL;563 }564 565 int Rc;566 if (NumDims == 1) {567 Rc = omp_target_memcpy(Dst, Src, ElementSize * Volume[0],568 ElementSize * DstOffsets[0],569 ElementSize * SrcOffsets[0], DstDevice, SrcDevice);570 } else {571 size_t DstSliceSize = ElementSize;572 size_t SrcSliceSize = ElementSize;573 for (int I = 1; I < NumDims; ++I) {574 DstSliceSize *= DstDimensions[I];575 SrcSliceSize *= SrcDimensions[I];576 }577 578 size_t DstOff = DstOffsets[0] * DstSliceSize;579 size_t SrcOff = SrcOffsets[0] * SrcSliceSize;580 for (size_t I = 0; I < Volume[0]; ++I) {581 Rc = omp_target_memcpy_rect(582 (char *)Dst + DstOff + DstSliceSize * I,583 (char *)const_cast<void *>(Src) + SrcOff + SrcSliceSize * I,584 ElementSize, NumDims - 1, Volume + 1, DstOffsets + 1, SrcOffsets + 1,585 DstDimensions + 1, SrcDimensions + 1, DstDevice, SrcDevice);586 587 if (Rc) {588 DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");589 return Rc;590 }591 }592 }593 594 DP("omp_target_memcpy_rect returns %d\n", Rc);595 return Rc;596}597 598EXTERN int omp_target_memcpy_rect_async(599 void *Dst, const void *Src, size_t ElementSize, int NumDims,600 const size_t *Volume, const size_t *DstOffsets, const size_t *SrcOffsets,601 const size_t *DstDimensions, const size_t *SrcDimensions, int DstDevice,602 int SrcDevice, int DepObjCount, omp_depend_t *DepObjList) {603 TIMESCOPE_WITH_DETAILS("dst_dev=" + std::to_string(DstDevice) +604 ";src_dev=" + std::to_string(SrcDevice) +605 ";size=" + std::to_string(ElementSize) +606 ";num_dims=" + std::to_string(NumDims));607 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));608 DP("Call to omp_target_memcpy_rect_async, dst device %d, src device %d, "609 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "610 "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "611 "volume " DPxMOD ", element size %zu, num_dims %d\n",612 DstDevice, SrcDevice, DPxPTR(Dst), DPxPTR(Src), DPxPTR(DstOffsets),613 DPxPTR(SrcOffsets), DPxPTR(DstDimensions), DPxPTR(SrcDimensions),614 DPxPTR(Volume), ElementSize, NumDims);615 616 // Need to check this first to not return OFFLOAD_FAIL instead617 if (!Dst && !Src) {618 DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",619 INT_MAX);620 return INT_MAX;621 }622 623 // Check the source and dest address624 if (Dst == nullptr || Src == nullptr)625 return OFFLOAD_FAIL;626 627 // Create task object628 TargetMemcpyArgsTy *Args = new TargetMemcpyArgsTy(629 Dst, Src, ElementSize, NumDims, Volume, DstOffsets, SrcOffsets,630 DstDimensions, SrcDimensions, DstDevice, SrcDevice);631 632 // Create and launch helper task633 int Rc = libomp_helper_task_creation(Args, &libomp_target_memcpy_async_task,634 DepObjCount, DepObjList);635 636 DP("omp_target_memcpy_rect_async returns %d\n", Rc);637 return Rc;638}639 640EXTERN int omp_target_associate_ptr(const void *HostPtr, const void *DevicePtr,641 size_t Size, size_t DeviceOffset,642 int DeviceNum) {643 TIMESCOPE();644 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));645 DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "646 "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",647 DPxPTR(HostPtr), DPxPTR(DevicePtr), Size, DeviceOffset, DeviceNum);648 649 if (!HostPtr || !DevicePtr || Size <= 0) {650 REPORT("Call to omp_target_associate_ptr with invalid arguments\n");651 return OFFLOAD_FAIL;652 }653 654 if (DeviceNum == omp_get_initial_device()) {655 REPORT("omp_target_associate_ptr: no association possible on the host\n");656 return OFFLOAD_FAIL;657 }658 659 auto DeviceOrErr = PM->getDevice(DeviceNum);660 if (!DeviceOrErr)661 FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());662 663 void *DeviceAddr = (void *)((uint64_t)DevicePtr + (uint64_t)DeviceOffset);664 665 OMPT_IF_BUILT(InterfaceRAII(666 RegionInterface.getCallbacks<ompt_target_data_associate>(), DeviceNum,667 const_cast<void *>(HostPtr), const_cast<void *>(DevicePtr), Size,668 __builtin_return_address(0)));669 670 int Rc = DeviceOrErr->getMappingInfo().associatePtr(671 const_cast<void *>(HostPtr), const_cast<void *>(DeviceAddr), Size);672 DP("omp_target_associate_ptr returns %d\n", Rc);673 return Rc;674}675 676EXTERN int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum) {677 TIMESCOPE();678 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));679 DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "680 "device_num %d\n",681 DPxPTR(HostPtr), DeviceNum);682 683 if (!HostPtr) {684 REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n");685 return OFFLOAD_FAIL;686 }687 688 if (DeviceNum == omp_get_initial_device()) {689 REPORT(690 "omp_target_disassociate_ptr: no association possible on the host\n");691 return OFFLOAD_FAIL;692 }693 694 auto DeviceOrErr = PM->getDevice(DeviceNum);695 if (!DeviceOrErr)696 FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());697 698 OMPT_IF_BUILT(InterfaceRAII(699 RegionInterface.getCallbacks<ompt_target_data_disassociate>(), DeviceNum,700 const_cast<void *>(HostPtr),701 /*DevicePtr=*/nullptr, /*Size=*/0, __builtin_return_address(0)));702 703 int Rc = DeviceOrErr->getMappingInfo().disassociatePtr(704 const_cast<void *>(HostPtr));705 DP("omp_target_disassociate_ptr returns %d\n", Rc);706 return Rc;707}708 709EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) {710 TIMESCOPE();711 OMPT_IF_BUILT(ReturnAddressSetterRAII RA(__builtin_return_address(0)));712 DP("Call to omp_get_mapped_ptr with ptr " DPxMOD ", device_num %d.\n",713 DPxPTR(Ptr), DeviceNum);714 715 if (!Ptr) {716 REPORT("Call to omp_get_mapped_ptr with nullptr.\n");717 return nullptr;718 }719 720 int NumDevices = omp_get_initial_device();721 if (DeviceNum == NumDevices) {722 DP("Device %d is initial device, returning Ptr " DPxMOD ".\n", DeviceNum,723 DPxPTR(Ptr));724 return const_cast<void *>(Ptr);725 }726 727 if (NumDevices <= DeviceNum) {728 DP("DeviceNum %d is invalid, returning nullptr.\n", DeviceNum);729 return nullptr;730 }731 732 auto DeviceOrErr = PM->getDevice(DeviceNum);733 if (!DeviceOrErr)734 FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());735 736 TargetPointerResultTy TPR =737 DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,738 /*UpdateRefCount=*/false,739 /*UseHoldRefCount=*/false);740 if (!TPR.isPresent()) {741 DP("Ptr " DPxMOD "is not present on device %d, returning nullptr.\n",742 DPxPTR(Ptr), DeviceNum);743 return nullptr;744 }745 746 DP("omp_get_mapped_ptr returns " DPxMOD ".\n", DPxPTR(TPR.TargetPointer));747 748 return TPR.TargetPointer;749}750 751// This routine gets called from the Host RTL at sync points (taskwait, barrier,752// ...) so we can synchronize the necessary objects from the offload side.753EXTERN void __tgt_target_sync(ident_t *loc_ref, int gtid, void *current_task,754 void *event) {755 if (!RTLAlive)756 return;757 758 RTLOngoingSyncs++;759 if (!RTLAlive) {760 RTLOngoingSyncs--;761 return;762 }763 764 syncImplicitInterops(gtid, event);765 766 RTLOngoingSyncs--;767}768