496 lines · cpp
1//===-- InteropAPI.cpp - Implementation of OpenMP interoperability API ----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "OpenMP/InteropAPI.h"10#include "OpenMP/InternalTypes.h"11#include "OpenMP/omp.h"12 13#include "OffloadPolicy.h"14#include "PluginManager.h"15#include "device.h"16#include "omptarget.h"17#include "llvm/Support/Error.h"18#include <cstdlib>19#include <cstring>20 21namespace {22omp_interop_rc_t getPropertyErrorType(omp_interop_property_t Property) {23 switch (Property) {24 case omp_ipr_fr_id:25 return omp_irc_type_int;26 case omp_ipr_fr_name:27 return omp_irc_type_str;28 case omp_ipr_vendor:29 return omp_irc_type_int;30 case omp_ipr_vendor_name:31 return omp_irc_type_str;32 case omp_ipr_device_num:33 return omp_irc_type_int;34 case omp_ipr_platform:35 return omp_irc_type_int;36 case omp_ipr_device:37 return omp_irc_type_ptr;38 case omp_ipr_device_context:39 return omp_irc_type_ptr;40 case omp_ipr_targetsync:41 return omp_irc_type_ptr;42 };43 return omp_irc_no_value;44}45 46void getTypeMismatch(omp_interop_property_t Property, int *Err) {47 if (Err)48 *Err = getPropertyErrorType(Property);49}50 51static const char *VendorStrTbl[] = {52 "unknown", "amd", "arm", "bsc", "fujitsu", "gnu", "hpe",53 "ibm", "intel", "llvm", "nec", "nvidia", "ti"};54const char *getVendorIdToStr(const omp_vendor_id_t VendorId) {55 if (VendorId < omp_vendor_unknown || VendorId >= omp_vendor_last)56 return ("unknown");57 return VendorStrTbl[VendorId];58}59 60static const char *ForeignRuntimeStrTbl[] = {61 "none", "cuda", "cuda_driver", "opencl",62 "sycl", "hip", "level_zero", "hsa"};63const char *getForeignRuntimeIdToStr(const tgt_foreign_runtime_id_t FrId) {64 if (FrId < tgt_fr_none || FrId >= tgt_fr_last)65 return ("unknown");66 return ForeignRuntimeStrTbl[FrId];67}68 69template <typename PropertyTy>70PropertyTy getProperty(omp_interop_val_t &InteropVal,71 omp_interop_property_t Property, int *Err);72 73template <>74intptr_t getProperty<intptr_t>(omp_interop_val_t &InteropVal,75 omp_interop_property_t Property, int *Err) {76 switch (Property) {77 case omp_ipr_fr_id:78 return InteropVal.fr_id;79 case omp_ipr_vendor:80 return InteropVal.vendor_id;81 case omp_ipr_device_num:82 return InteropVal.device_id;83 default:;84 }85 getTypeMismatch(Property, Err);86 return 0;87}88 89template <>90const char *getProperty<const char *>(omp_interop_val_t &InteropVal,91 omp_interop_property_t Property,92 int *Err) {93 switch (Property) {94 case omp_ipr_fr_name:95 return getForeignRuntimeIdToStr(InteropVal.fr_id);96 case omp_ipr_vendor_name:97 return getVendorIdToStr(InteropVal.vendor_id);98 default:99 getTypeMismatch(Property, Err);100 return nullptr;101 }102}103 104template <>105void *getProperty<void *>(omp_interop_val_t &InteropVal,106 omp_interop_property_t Property, int *Err) {107 switch (Property) {108 case omp_ipr_device:109 if (InteropVal.device_info.Device)110 return InteropVal.device_info.Device;111 *Err = omp_irc_no_value;112 return const_cast<char *>(InteropVal.err_str);113 case omp_ipr_platform:114 return InteropVal.device_info.Platform;115 case omp_ipr_device_context:116 return InteropVal.device_info.Context;117 case omp_ipr_targetsync:118 return InteropVal.async_info ? InteropVal.async_info->Queue : nullptr;119 default:;120 }121 getTypeMismatch(Property, Err);122 return nullptr;123}124 125bool getPropertyCheck(omp_interop_val_t **InteropPtr,126 omp_interop_property_t Property, int *Err) {127 if (Err)128 *Err = omp_irc_success;129 if (!InteropPtr) {130 if (Err)131 *Err = omp_irc_empty;132 return false;133 }134 if (Property >= 0 || Property < omp_ipr_first) {135 if (Err)136 *Err = omp_irc_out_of_range;137 return false;138 }139 if (Property == omp_ipr_targetsync &&140 (*InteropPtr)->interop_type != kmp_interop_type_targetsync) {141 if (Err)142 *Err = omp_irc_other;143 return false;144 }145 if ((Property == omp_ipr_device || Property == omp_ipr_device_context) &&146 (*InteropPtr)->interop_type == kmp_interop_type_targetsync) {147 if (Err)148 *Err = omp_irc_other;149 return false;150 }151 return true;152}153 154} // namespace155 156#define __OMP_GET_INTEROP_TY(RETURN_TYPE, SUFFIX) \157 RETURN_TYPE omp_get_interop_##SUFFIX(const omp_interop_t interop, \158 omp_interop_property_t property_id, \159 int *err) { \160 omp_interop_val_t *interop_val = (omp_interop_val_t *)interop; \161 if (!getPropertyCheck(&interop_val, property_id, err)) { \162 return (RETURN_TYPE)(0); \163 } \164 return getProperty<RETURN_TYPE>(*interop_val, property_id, err); \165 }166__OMP_GET_INTEROP_TY(intptr_t, int)167__OMP_GET_INTEROP_TY(void *, ptr)168__OMP_GET_INTEROP_TY(const char *, str)169#undef __OMP_GET_INTEROP_TY170 171#define __OMP_GET_INTEROP_TY3(RETURN_TYPE, SUFFIX) \172 RETURN_TYPE omp_get_interop_##SUFFIX(const omp_interop_t interop, \173 omp_interop_property_t property_id) { \174 int err; \175 omp_interop_val_t *interop_val = (omp_interop_val_t *)interop; \176 if (!getPropertyCheck(&interop_val, property_id, &err)) { \177 return (RETURN_TYPE)(0); \178 } \179 return nullptr; \180 return getProperty<RETURN_TYPE>(*interop_val, property_id, &err); \181 }182__OMP_GET_INTEROP_TY3(const char *, name)183__OMP_GET_INTEROP_TY3(const char *, type_desc)184__OMP_GET_INTEROP_TY3(const char *, rc_desc)185#undef __OMP_GET_INTEROP_TY3186 187extern "C" {188 189omp_interop_val_t *__tgt_interop_get(ident_t *LocRef, int32_t InteropType,190 int64_t DeviceNum, int32_t NumPrefers,191 interop_spec_t *Prefers,192 interop_ctx_t *Ctx, dep_pack_t *Deps) {193 194 DP("Call to %s with device_num %" PRId64 ", interop type %" PRId32195 ", number of preferred specs %" PRId32 "%s%s\n",196 __func__, DeviceNum, InteropType, NumPrefers,197 Ctx->flags.implicit ? " (implicit)" : "",198 Ctx->flags.nowait ? " (nowait)" : "");199 200 if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED)201 return omp_interop_none;202 203 // Now, try to create an interop with device_num.204 if (DeviceNum == OFFLOAD_DEVICE_DEFAULT)205 DeviceNum = omp_get_default_device();206 207 auto gtid = Ctx->gtid;208 209 if (InteropType == kmp_interop_type_targetsync) {210 if (Ctx->flags.nowait)211 DP("Warning: nowait flag on interop creation not supported yet. "212 "Ignored\n");213 if (Deps)214 __kmpc_omp_wait_deps(LocRef, gtid, Deps->ndeps, Deps->deplist,215 Deps->ndeps_noalias, Deps->noalias_deplist);216 }217 218 auto DeviceOrErr = PM->getDevice(DeviceNum);219 if (!DeviceOrErr) {220 DP("Couldn't find device %" PRId64221 " while constructing interop object: %s\n",222 DeviceNum, toString(DeviceOrErr.takeError()).c_str());223 return omp_interop_none;224 }225 auto &Device = *DeviceOrErr;226 omp_interop_val_t *Interop = omp_interop_none;227 auto InteropSpec = Device.RTL->select_interop_preference(228 DeviceNum, InteropType, NumPrefers, Prefers);229 if (InteropSpec.fr_id == tgt_fr_none) {230 DP("Interop request not supported by device %" PRId64 "\n", DeviceNum);231 return omp_interop_none;232 }233 DP("Selected interop preference is fr_id=%s%s impl_attrs=%" PRId64 "\n",234 getForeignRuntimeIdToStr((tgt_foreign_runtime_id_t)InteropSpec.fr_id),235 InteropSpec.attrs.inorder ? " inorder" : "", InteropSpec.impl_attrs);236 237 if (Ctx->flags.implicit) {238 // This is a request for an RTL managed interop object.239 // Get it from the InteropTbl if possible240 for (auto iop : PM->InteropTbl) {241 if (iop->isCompatibleWith(InteropType, InteropSpec, DeviceNum, gtid)) {242 Interop = iop;243 Interop->markDirty();244 DP("Reused interop " DPxMOD " from device number %" PRId64245 " for gtid %" PRId32 "\n",246 DPxPTR(Interop), DeviceNum, gtid);247 return Interop;248 }249 }250 }251 252 Interop = Device.RTL->create_interop(DeviceNum, InteropType, &InteropSpec);253 DP("Created an interop " DPxMOD " from device number %" PRId64 "\n",254 DPxPTR(Interop), DeviceNum);255 256 if (Ctx->flags.implicit) {257 // register the new implicit interop in the RTL258 Interop->setOwner(gtid);259 Interop->markDirty();260 PM->InteropTbl.add(Interop);261 } else {262 Interop->setOwner(omp_interop_val_t::no_owner);263 }264 265 return Interop;266}267 268int __tgt_interop_use60(ident_t *LocRef, omp_interop_val_t *Interop,269 interop_ctx_t *Ctx, dep_pack_t *Deps) {270 bool Nowait = Ctx->flags.nowait;271 DP("Call to %s with interop " DPxMOD ", nowait %" PRId32 "\n", __func__,272 DPxPTR(Interop), Nowait);273 if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED || !Interop)274 return OFFLOAD_FAIL;275 276 if (Interop->interop_type == kmp_interop_type_targetsync) {277 if (Deps) {278 if (Nowait) {279 DP("Warning: nowait flag on interop use with dependences not supported"280 "yet. Ignored\n");281 Nowait = false;282 }283 284 __kmpc_omp_wait_deps(LocRef, Ctx->gtid, Deps->ndeps, Deps->deplist,285 Deps->ndeps_noalias, Deps->noalias_deplist);286 }287 }288 289 auto DeviceOrErr = Interop->getDevice();290 if (!DeviceOrErr) {291 REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(Interop),292 toString(DeviceOrErr.takeError()).c_str());293 return OFFLOAD_FAIL;294 }295 auto &IOPDevice = *DeviceOrErr;296 297 if (Interop->async_info && Interop->async_info->Queue) {298 if (Nowait)299 Interop->async_barrier(IOPDevice);300 else {301 Interop->flush(IOPDevice);302 Interop->sync_barrier(IOPDevice);303 Interop->markClean();304 }305 }306 307 return OFFLOAD_SUCCESS;308}309 310int __tgt_interop_release(ident_t *LocRef, omp_interop_val_t *Interop,311 interop_ctx_t *Ctx, dep_pack_t *Deps) {312 DP("Call to %s with interop " DPxMOD "\n", __func__, DPxPTR(Interop));313 314 if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED || !Interop)315 return OFFLOAD_FAIL;316 317 if (Interop->interop_type == kmp_interop_type_targetsync) {318 if (Ctx->flags.nowait)319 DP("Warning: nowait flag on interop destroy not supported "320 "yet. Ignored\n");321 if (Deps) {322 __kmpc_omp_wait_deps(LocRef, Ctx->gtid, Deps->ndeps, Deps->deplist,323 Deps->ndeps_noalias, Deps->noalias_deplist);324 }325 }326 327 auto DeviceOrErr = Interop->getDevice();328 if (!DeviceOrErr) {329 REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(Interop),330 toString(DeviceOrErr.takeError()).c_str());331 return OFFLOAD_FAIL;332 }333 334 return Interop->release(*DeviceOrErr);335}336 337EXTERN int ompx_interop_add_completion_callback(omp_interop_val_t *Interop,338 ompx_interop_cb_t *CB,339 void *Data) {340 DP("Call to %s with interop " DPxMOD ", property callback " DPxMOD341 "and data " DPxMOD "\n",342 __func__, DPxPTR(Interop), DPxPTR(CB), DPxPTR(Data));343 344 if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED || !Interop)345 return omp_irc_other;346 347 Interop->addCompletionCb(CB, Data);348 349 return omp_irc_success;350}351 352// Backwards compatibility wrappers353void __tgt_interop_init(ident_t *LocRef, int32_t Gtid,354 omp_interop_val_t *&InteropPtr, int32_t InteropType,355 int32_t DeviceId, int32_t Ndeps,356 kmp_depend_info_t *DepList, int32_t HaveNowait) {357 constexpr int32_t old_kmp_interop_type_targetsync = 2;358 interop_ctx_t Ctx = {0, {false, (bool)HaveNowait, 0}, Gtid};359 dep_pack_t Deps = {Ndeps, 0, DepList, nullptr};360 InteropPtr =361 __tgt_interop_get(LocRef,362 InteropType == old_kmp_interop_type_targetsync363 ? kmp_interop_type_targetsync364 : kmp_interop_type_target,365 DeviceId, 0, nullptr, &Ctx, Ndeps ? &Deps : nullptr);366}367 368void __tgt_interop_use(ident_t *LocRef, int32_t Gtid,369 omp_interop_val_t *&InteropPtr, int32_t DeviceId,370 int32_t Ndeps, kmp_depend_info_t *DepList,371 int32_t HaveNowait) {372 interop_ctx_t Ctx = {0, {false, (bool)HaveNowait, 0}, Gtid};373 dep_pack_t Deps = {Ndeps, 0, DepList, nullptr};374 __tgt_interop_use60(LocRef, InteropPtr, &Ctx, Ndeps ? &Deps : nullptr);375}376 377void __tgt_interop_destroy(ident_t *LocRef, int32_t Gtid,378 omp_interop_val_t *&InteropPtr, int32_t DeviceId,379 int32_t Ndeps, kmp_depend_info_t *DepList,380 int32_t HaveNowait) {381 interop_ctx_t Ctx = {0, {false, (bool)HaveNowait, 0}, Gtid};382 dep_pack_t Deps = {Ndeps, 0, DepList, nullptr};383 __tgt_interop_release(LocRef, InteropPtr, &Ctx, Ndeps ? &Deps : nullptr);384}385 386} // extern "C"387 388llvm::Expected<DeviceTy &> omp_interop_val_t::getDevice() const {389 return PM->getDevice(device_id);390}391 392bool omp_interop_val_t::isCompatibleWith(int32_t InteropType,393 const interop_spec_t &Spec) {394 if (interop_type != InteropType)395 return false;396 if (Spec.fr_id != fr_id)397 return false;398 if (Spec.attrs.inorder != attrs.inorder)399 return false;400 if (Spec.impl_attrs != impl_attrs)401 return false;402 403 return true;404}405 406bool omp_interop_val_t::isCompatibleWith(int32_t InteropType,407 const interop_spec_t &Spec,408 int64_t DeviceNum, int GTID) {409 if (device_id != DeviceNum)410 return false;411 412 if (GTID != owner_gtid)413 return false;414 415 return isCompatibleWith(InteropType, Spec);416}417 418int32_t omp_interop_val_t::flush(DeviceTy &Device) {419 return Device.RTL->flush_queue(this);420}421 422int32_t omp_interop_val_t::sync_barrier(DeviceTy &Device) {423 if (Device.RTL->sync_barrier(this) != OFFLOAD_SUCCESS) {424 FATAL_MESSAGE(device_id, "Interop sync barrier failed for %p object\n",425 this);426 }427 DP("Calling completion callbacks for " DPxMOD "\n", DPxPTR(this));428 runCompletionCbs();429 return OFFLOAD_SUCCESS;430}431 432int32_t omp_interop_val_t::async_barrier(DeviceTy &Device) {433 return Device.RTL->async_barrier(this);434}435 436int32_t omp_interop_val_t::release(DeviceTy &Device) {437 if (async_info != nullptr && (!hasOwner() || !isClean())) {438 flush(Device);439 sync_barrier(Device);440 }441 return Device.RTL->release_interop(device_id, this);442}443 444void syncImplicitInterops(int Gtid, void *Event) {445 if (PM->InteropTbl.size() == 0)446 return;447 448 DP("target_sync: syncing interops for gtid %" PRId32 ", event " DPxMOD "\n",449 Gtid, DPxPTR(Event));450 451 for (auto iop : PM->InteropTbl) {452 if (iop->async_info && iop->async_info->Queue && iop->isOwnedBy(Gtid) &&453 !iop->isClean()) {454 455 auto DeviceOrErr = iop->getDevice();456 if (!DeviceOrErr) {457 REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(iop),458 toString(DeviceOrErr.takeError()).c_str());459 continue;460 }461 auto &IOPDevice = *DeviceOrErr;462 463 iop->flush(IOPDevice);464 iop->sync_barrier(IOPDevice);465 iop->markClean();466 467 // Alternate implementation option in case using barriers is not468 // efficient enough:469 //470 // Instead of using a synchronous barrier, queue an asynchronous471 // barrier and create a proxy task associated to the event to handle472 // OpenMP synchronizations.473 // When the event is completed, fulfill the proxy task to notify the474 // OpenMP runtime.475 // event = iop->asyncBarrier();476 // ptask = createProxyTask();477 // Events->add(event,ptask);478 }479 }480 // This would be needed for the alternate implementation481 // processEvents();482}483 484void InteropTblTy::clear() {485 DP("Clearing Interop Table\n");486 PerThreadTable::clear([](auto &IOP) {487 auto DeviceOrErr = IOP->getDevice();488 if (!DeviceOrErr) {489 REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(IOP),490 toString(DeviceOrErr.takeError()).c_str());491 return;492 }493 IOP->release(*DeviceOrErr);494 });495}496