1219 lines · cpp
1//===- ol_impl.cpp - Implementation of the new LLVM/Offload API ------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains the definitions of the new LLVM/Offload API entry points. See10// new-api/API/README.md for more information.11//12//===----------------------------------------------------------------------===//13 14#include "OffloadImpl.hpp"15#include "Helpers.hpp"16#include "OffloadPrint.hpp"17#include "PluginManager.h"18#include "llvm/Support/FormatVariadic.h"19#include <OffloadAPI.h>20 21#include <mutex>22 23// TODO: Some plugins expect to be linked into libomptarget which defines these24// symbols to implement ompt callbacks. The least invasive workaround here is to25// define them in libLLVMOffload as false/null so they are never used. In future26// it would be better to allow the plugins to implement callbacks without27// pulling in details from libomptarget.28#ifdef OMPT_SUPPORT29namespace llvm::omp::target {30namespace ompt {31bool Initialized = false;32ompt_get_callback_t lookupCallbackByCode = nullptr;33ompt_function_lookup_t lookupCallbackByName = nullptr;34} // namespace ompt35} // namespace llvm::omp::target36#endif37 38using namespace llvm::omp::target;39using namespace llvm::omp::target::plugin;40using namespace error;41 42struct ol_platform_impl_t {43 ol_platform_impl_t(std::unique_ptr<GenericPluginTy> Plugin,44 ol_platform_backend_t BackendType)45 : BackendType(BackendType), Plugin(std::move(Plugin)) {}46 ol_platform_backend_t BackendType;47 48 /// Complete all pending work for this platform and perform any needed49 /// cleanup.50 ///51 /// After calling this function, no liboffload functions should be called with52 /// this platform handle.53 llvm::Error destroy();54 55 /// Initialize the associated plugin and devices.56 llvm::Error init();57 58 /// Direct access to the plugin, may be uninitialized if accessed here.59 std::unique_ptr<GenericPluginTy> Plugin;60 61 llvm::SmallVector<std::unique_ptr<ol_device_impl_t>> Devices;62};63 64// Handle type definitions. Ideally these would be 1:1 with the plugins, but65// we add some additional data here for now to avoid churn in the plugin66// interface.67struct ol_device_impl_t {68 ol_device_impl_t(int DeviceNum, GenericDeviceTy *Device,69 ol_platform_impl_t &Platform, InfoTreeNode &&DevInfo)70 : DeviceNum(DeviceNum), Device(Device), Platform(Platform),71 Info(std::forward<InfoTreeNode>(DevInfo)) {}72 73 ~ol_device_impl_t() {74 assert(!OutstandingQueues.size() &&75 "Device object dropped with outstanding queues");76 }77 78 int DeviceNum;79 GenericDeviceTy *Device;80 ol_platform_impl_t &Platform;81 InfoTreeNode Info;82 83 llvm::SmallVector<__tgt_async_info *> OutstandingQueues;84 std::mutex OutstandingQueuesMutex;85 86 /// If the device has any outstanding queues that are now complete, remove it87 /// from the list and return it.88 ///89 /// Queues may be added to the outstanding queue list by olDestroyQueue if90 /// they are destroyed but not completed.91 __tgt_async_info *getOutstandingQueue() {92 // Not locking the `size()` access is fine here - In the worst case we93 // either miss a queue that exists or loop through an empty array after94 // taking the lock. Both are sub-optimal but not that bad.95 if (OutstandingQueues.size()) {96 std::lock_guard<std::mutex> Lock(OutstandingQueuesMutex);97 98 // As queues are pulled and popped from this list, longer running queues99 // naturally bubble to the start of the array. Hence looping backwards.100 for (auto Q = OutstandingQueues.rbegin(); Q != OutstandingQueues.rend();101 Q++) {102 if (!Device->hasPendingWork(*Q)) {103 auto OutstandingQueue = *Q;104 *Q = OutstandingQueues.back();105 OutstandingQueues.pop_back();106 return OutstandingQueue;107 }108 }109 }110 return nullptr;111 }112 113 /// Complete all pending work for this device and perform any needed cleanup.114 ///115 /// After calling this function, no liboffload functions should be called with116 /// this device handle.117 llvm::Error destroy() {118 llvm::Error Result = Plugin::success();119 for (auto Q : OutstandingQueues)120 if (auto Err = Device->synchronize(Q, /*Release=*/true))121 Result = llvm::joinErrors(std::move(Result), std::move(Err));122 OutstandingQueues.clear();123 return Result;124 }125};126 127llvm::Error ol_platform_impl_t::destroy() {128 llvm::Error Result = Plugin::success();129 for (auto &D : Devices)130 if (auto Err = D->destroy())131 Result = llvm::joinErrors(std::move(Result), std::move(Err));132 133 if (auto Res = Plugin->deinit())134 Result = llvm::joinErrors(std::move(Result), std::move(Res));135 136 return Result;137}138 139llvm::Error ol_platform_impl_t::init() {140 if (!Plugin)141 return llvm::Error::success();142 143 if (llvm::Error Err = Plugin->init())144 return Err;145 146 for (auto Id = 0, End = Plugin->getNumDevices(); Id != End; Id++) {147 if (llvm::Error Err = Plugin->initDevice(Id))148 return Err;149 150 GenericDeviceTy *Device = &Plugin->getDevice(Id);151 llvm::Expected<InfoTreeNode> Info = Device->obtainInfo();152 if (llvm::Error Err = Info.takeError())153 return Err;154 Devices.emplace_back(std::make_unique<ol_device_impl_t>(Id, Device, *this,155 std::move(*Info)));156 }157 158 return llvm::Error::success();159}160 161struct ol_queue_impl_t {162 ol_queue_impl_t(__tgt_async_info *AsyncInfo, ol_device_handle_t Device)163 : AsyncInfo(AsyncInfo), Device(Device), Id(IdCounter++) {}164 __tgt_async_info *AsyncInfo;165 ol_device_handle_t Device;166 // A unique identifier for the queue167 size_t Id;168 static std::atomic<size_t> IdCounter;169};170std::atomic<size_t> ol_queue_impl_t::IdCounter(0);171 172struct ol_event_impl_t {173 ol_event_impl_t(void *EventInfo, ol_device_handle_t Device,174 ol_queue_handle_t Queue)175 : EventInfo(EventInfo), Device(Device), QueueId(Queue->Id), Queue(Queue) {176 }177 // EventInfo may be null, in which case the event should be considered always178 // complete179 void *EventInfo;180 ol_device_handle_t Device;181 size_t QueueId;182 // Events may outlive the queue - don't assume this is always valid.183 // It is provided only to implement OL_EVENT_INFO_QUEUE. Use QueueId to check184 // for queue equality instead.185 ol_queue_handle_t Queue;186};187 188struct ol_program_impl_t {189 ol_program_impl_t(plugin::DeviceImageTy *Image,190 llvm::MemoryBufferRef DeviceImage)191 : Image(Image), DeviceImage(DeviceImage) {}192 plugin::DeviceImageTy *Image;193 std::mutex SymbolListMutex;194 llvm::MemoryBufferRef DeviceImage;195 llvm::StringMap<std::unique_ptr<ol_symbol_impl_t>> KernelSymbols;196 llvm::StringMap<std::unique_ptr<ol_symbol_impl_t>> GlobalSymbols;197};198 199struct ol_symbol_impl_t {200 ol_symbol_impl_t(const char *Name, GenericKernelTy *Kernel)201 : PluginImpl(Kernel), Kind(OL_SYMBOL_KIND_KERNEL), Name(Name) {}202 ol_symbol_impl_t(const char *Name, GlobalTy &&Global)203 : PluginImpl(Global), Kind(OL_SYMBOL_KIND_GLOBAL_VARIABLE), Name(Name) {}204 std::variant<GenericKernelTy *, GlobalTy> PluginImpl;205 ol_symbol_kind_t Kind;206 llvm::StringRef Name;207};208 209namespace llvm {210namespace offload {211 212struct AllocInfo {213 ol_device_handle_t Device;214 ol_alloc_type_t Type;215 void *Start;216 // One byte past the end217 void *End;218};219 220// Global shared state for liboffload221struct OffloadContext;222// This pointer is non-null if and only if the context is valid and fully223// initialized224static std::atomic<OffloadContext *> OffloadContextVal;225std::mutex OffloadContextValMutex;226struct OffloadContext {227 OffloadContext(OffloadContext &) = delete;228 OffloadContext(OffloadContext &&) = delete;229 OffloadContext &operator=(OffloadContext &) = delete;230 OffloadContext &operator=(OffloadContext &&) = delete;231 232 bool TracingEnabled = false;233 bool ValidationEnabled = true;234 DenseMap<void *, AllocInfo> AllocInfoMap{};235 std::mutex AllocInfoMapMutex{};236 // Partitioned list of memory base addresses. Each element in this list is a237 // key in AllocInfoMap238 SmallVector<void *> AllocBases{};239 SmallVector<std::unique_ptr<ol_platform_impl_t>, 4> Platforms{};240 ol_device_handle_t HostDevice;241 size_t RefCount;242 243 static OffloadContext &get() {244 assert(OffloadContextVal);245 return *OffloadContextVal;246 }247};248 249// If the context is uninited, then we assume tracing is disabled250bool isTracingEnabled() {251 return isOffloadInitialized() && OffloadContext::get().TracingEnabled;252}253bool isValidationEnabled() { return OffloadContext::get().ValidationEnabled; }254bool isOffloadInitialized() { return OffloadContextVal != nullptr; }255 256template <typename HandleT> Error olDestroy(HandleT Handle) {257 delete Handle;258 return Error::success();259}260 261constexpr ol_platform_backend_t pluginNameToBackend(StringRef Name) {262 if (Name == "amdgpu") {263 return OL_PLATFORM_BACKEND_AMDGPU;264 } else if (Name == "cuda") {265 return OL_PLATFORM_BACKEND_CUDA;266 } else {267 return OL_PLATFORM_BACKEND_UNKNOWN;268 }269}270 271// Every plugin exports this method to create an instance of the plugin type.272#define PLUGIN_TARGET(Name) extern "C" GenericPluginTy *createPlugin_##Name();273#include "Shared/Targets.def"274 275Error initPlugins(OffloadContext &Context) {276 // Attempt to create an instance of each supported plugin.277#define PLUGIN_TARGET(Name) \278 do { \279 if (StringRef(#Name) != "host") \280 Context.Platforms.emplace_back(std::make_unique<ol_platform_impl_t>( \281 std::unique_ptr<GenericPluginTy>(createPlugin_##Name()), \282 pluginNameToBackend(#Name))); \283 } while (false);284#include "Shared/Targets.def"285 286 // Eagerly initialize all of the plugins and devices. We need to make sure287 // that the platform is initialized at a consistent point to maintain the288 // expected teardown order in the vendor libraries.289 for (auto &Platform : Context.Platforms) {290 if (Error Err = Platform->init())291 return Err;292 }293 294 // Add the special host device.295 auto &HostPlatform = Context.Platforms.emplace_back(296 std::make_unique<ol_platform_impl_t>(nullptr, OL_PLATFORM_BACKEND_HOST));297 Context.HostDevice = HostPlatform->Devices298 .emplace_back(std::make_unique<ol_device_impl_t>(299 -1, nullptr, *HostPlatform, InfoTreeNode{}))300 .get();301 302 Context.TracingEnabled = std::getenv("OFFLOAD_TRACE");303 Context.ValidationEnabled = !std::getenv("OFFLOAD_DISABLE_VALIDATION");304 305 return Plugin::success();306}307 308Error olInit_impl() {309 std::lock_guard<std::mutex> Lock(OffloadContextValMutex);310 311 if (isOffloadInitialized()) {312 OffloadContext::get().RefCount++;313 return Plugin::success();314 }315 316 // Use a temporary to ensure that entry points querying OffloadContextVal do317 // not get a partially initialized context318 auto *NewContext = new OffloadContext{};319 Error InitResult = initPlugins(*NewContext);320 OffloadContextVal.store(NewContext);321 OffloadContext::get().RefCount++;322 323 return InitResult;324}325 326Error olShutDown_impl() {327 std::lock_guard<std::mutex> Lock(OffloadContextValMutex);328 329 if (--OffloadContext::get().RefCount != 0)330 return Error::success();331 332 Error Result = Error::success();333 auto *OldContext = OffloadContextVal.exchange(nullptr);334 335 for (auto &Platform : OldContext->Platforms) {336 // Host plugin is nullptr and has no deinit337 if (!Platform->Plugin || !Platform->Plugin->is_initialized())338 continue;339 340 if (auto Res = Platform->destroy())341 Result = joinErrors(std::move(Result), std::move(Res));342 }343 344 delete OldContext;345 return Result;346}347 348Error olGetPlatformInfoImplDetail(ol_platform_handle_t Platform,349 ol_platform_info_t PropName, size_t PropSize,350 void *PropValue, size_t *PropSizeRet) {351 InfoWriter Info(PropSize, PropValue, PropSizeRet);352 bool IsHost = Platform->BackendType == OL_PLATFORM_BACKEND_HOST;353 354 // Note that the plugin is potentially uninitialized here. It will need to be355 // initialized once info is added that requires it to be initialized.356 switch (PropName) {357 case OL_PLATFORM_INFO_NAME:358 return Info.writeString(IsHost ? "Host" : Platform->Plugin->getName());359 case OL_PLATFORM_INFO_VENDOR_NAME:360 // TODO: Implement this361 return Info.writeString("Unknown platform vendor");362 case OL_PLATFORM_INFO_VERSION: {363 return Info.writeString(formatv("v{0}.{1}.{2}", OL_VERSION_MAJOR,364 OL_VERSION_MINOR, OL_VERSION_PATCH)365 .str());366 }367 case OL_PLATFORM_INFO_BACKEND: {368 return Info.write<ol_platform_backend_t>(Platform->BackendType);369 }370 default:371 return createOffloadError(ErrorCode::INVALID_ENUMERATION,372 "getPlatformInfo enum '%i' is invalid", PropName);373 }374 375 return Error::success();376}377 378Error olGetPlatformInfo_impl(ol_platform_handle_t Platform,379 ol_platform_info_t PropName, size_t PropSize,380 void *PropValue) {381 return olGetPlatformInfoImplDetail(Platform, PropName, PropSize, PropValue,382 nullptr);383}384 385Error olGetPlatformInfoSize_impl(ol_platform_handle_t Platform,386 ol_platform_info_t PropName,387 size_t *PropSizeRet) {388 return olGetPlatformInfoImplDetail(Platform, PropName, 0, nullptr,389 PropSizeRet);390}391 392Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,393 ol_device_info_t PropName, size_t PropSize,394 void *PropValue, size_t *PropSizeRet) {395 assert(Device != OffloadContext::get().HostDevice);396 InfoWriter Info(PropSize, PropValue, PropSizeRet);397 398 auto makeError = [&](ErrorCode Code, StringRef Err) {399 std::string ErrBuffer;400 raw_string_ostream(ErrBuffer) << PropName << ": " << Err;401 return Plugin::error(ErrorCode::UNIMPLEMENTED, ErrBuffer.c_str());402 };403 404 // These are not implemented by the plugin interface405 switch (PropName) {406 case OL_DEVICE_INFO_PLATFORM:407 return Info.write<void *>(&Device->Platform);408 409 case OL_DEVICE_INFO_TYPE:410 return Info.write<ol_device_type_t>(OL_DEVICE_TYPE_GPU);411 412 case OL_DEVICE_INFO_SINGLE_FP_CONFIG:413 case OL_DEVICE_INFO_DOUBLE_FP_CONFIG: {414 ol_device_fp_capability_flags_t flags{0};415 flags |= OL_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT |416 OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST |417 OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO |418 OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF |419 OL_DEVICE_FP_CAPABILITY_FLAG_INF_NAN |420 OL_DEVICE_FP_CAPABILITY_FLAG_DENORM |421 OL_DEVICE_FP_CAPABILITY_FLAG_FMA;422 return Info.write(flags);423 }424 425 case OL_DEVICE_INFO_HALF_FP_CONFIG:426 return Info.write<ol_device_fp_capability_flags_t>(0);427 428 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_CHAR:429 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_SHORT:430 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_INT:431 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG:432 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_FLOAT:433 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE:434 return Info.write<uint32_t>(1);435 436 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF:437 return Info.write<uint32_t>(0);438 439 // None of the existing plugins specify a limit on a single allocation,440 // so return the global memory size instead441 case OL_DEVICE_INFO_MAX_MEM_ALLOC_SIZE:442 [[fallthrough]];443 // AMD doesn't provide the global memory size (trivially) with the device info444 // struct, so use the plugin interface445 case OL_DEVICE_INFO_GLOBAL_MEM_SIZE: {446 uint64_t Mem;447 if (auto Err = Device->Device->getDeviceMemorySize(Mem))448 return Err;449 return Info.write<uint64_t>(Mem);450 } break;451 452 default:453 break;454 }455 456 if (PropName >= OL_DEVICE_INFO_LAST)457 return createOffloadError(ErrorCode::INVALID_ENUMERATION,458 "getDeviceInfo enum '%i' is invalid", PropName);459 460 auto EntryOpt = Device->Info.get(static_cast<DeviceInfo>(PropName));461 if (!EntryOpt)462 return makeError(ErrorCode::UNIMPLEMENTED,463 "plugin did not provide a response for this information");464 auto Entry = *EntryOpt;465 466 // Retrieve properties from the plugin interface467 switch (PropName) {468 case OL_DEVICE_INFO_NAME:469 case OL_DEVICE_INFO_PRODUCT_NAME:470 case OL_DEVICE_INFO_UID:471 case OL_DEVICE_INFO_VENDOR:472 case OL_DEVICE_INFO_DRIVER_VERSION: {473 // String values474 if (!std::holds_alternative<std::string>(Entry->Value))475 return makeError(ErrorCode::BACKEND_FAILURE,476 "plugin returned incorrect type");477 return Info.writeString(std::get<std::string>(Entry->Value).c_str());478 }479 480 case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE:481 case OL_DEVICE_INFO_MAX_WORK_SIZE:482 case OL_DEVICE_INFO_VENDOR_ID:483 case OL_DEVICE_INFO_NUM_COMPUTE_UNITS:484 case OL_DEVICE_INFO_ADDRESS_BITS:485 case OL_DEVICE_INFO_MAX_CLOCK_FREQUENCY:486 case OL_DEVICE_INFO_MEMORY_CLOCK_RATE: {487 // Uint32 values488 if (!std::holds_alternative<uint64_t>(Entry->Value))489 return makeError(ErrorCode::BACKEND_FAILURE,490 "plugin returned incorrect type");491 auto Value = std::get<uint64_t>(Entry->Value);492 if (Value > std::numeric_limits<uint32_t>::max())493 return makeError(ErrorCode::BACKEND_FAILURE,494 "plugin returned out of range device info");495 return Info.write(static_cast<uint32_t>(Value));496 }497 498 case OL_DEVICE_INFO_WORK_GROUP_LOCAL_MEM_SIZE: {499 if (!std::holds_alternative<uint64_t>(Entry->Value))500 return makeError(ErrorCode::BACKEND_FAILURE,501 "plugin returned incorrect type");502 return Info.write(std::get<uint64_t>(Entry->Value));503 }504 505 case OL_DEVICE_INFO_MAX_WORK_SIZE_PER_DIMENSION:506 case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE_PER_DIMENSION: {507 // {x, y, z} triples508 ol_dimensions_t Out{0, 0, 0};509 510 auto getField = [&](StringRef Name, uint32_t &Dest) {511 if (auto F = Entry->get(Name)) {512 if (!std::holds_alternative<size_t>((*F)->Value))513 return makeError(514 ErrorCode::BACKEND_FAILURE,515 "plugin returned incorrect type for dimensions element");516 Dest = std::get<size_t>((*F)->Value);517 } else518 return makeError(ErrorCode::BACKEND_FAILURE,519 "plugin didn't provide all values for dimensions");520 return Plugin::success();521 };522 523 if (auto Res = getField("x", Out.x))524 return Res;525 if (auto Res = getField("y", Out.y))526 return Res;527 if (auto Res = getField("z", Out.z))528 return Res;529 530 return Info.write(Out);531 }532 533 default:534 llvm_unreachable("Unimplemented device info");535 }536}537 538Error olGetDeviceInfoImplDetailHost(ol_device_handle_t Device,539 ol_device_info_t PropName, size_t PropSize,540 void *PropValue, size_t *PropSizeRet) {541 assert(Device == OffloadContext::get().HostDevice);542 InfoWriter Info(PropSize, PropValue, PropSizeRet);543 544 constexpr auto uint32_max = std::numeric_limits<uint32_t>::max();545 546 switch (PropName) {547 case OL_DEVICE_INFO_PLATFORM:548 return Info.write<void *>(&Device->Platform);549 case OL_DEVICE_INFO_TYPE:550 return Info.write<ol_device_type_t>(OL_DEVICE_TYPE_HOST);551 case OL_DEVICE_INFO_NAME:552 return Info.writeString("Virtual Host Device");553 case OL_DEVICE_INFO_PRODUCT_NAME:554 return Info.writeString("Virtual Host Device");555 case OL_DEVICE_INFO_UID:556 return Info.writeString(GenericPluginTy::getHostDeviceUid());557 case OL_DEVICE_INFO_VENDOR:558 return Info.writeString("Liboffload");559 case OL_DEVICE_INFO_DRIVER_VERSION:560 return Info.writeString(LLVM_VERSION_STRING);561 case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE:562 return Info.write<uint32_t>(1);563 case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE_PER_DIMENSION:564 return Info.write<ol_dimensions_t>(ol_dimensions_t{1, 1, 1});565 case OL_DEVICE_INFO_MAX_WORK_SIZE:566 return Info.write<uint32_t>(uint32_max);567 case OL_DEVICE_INFO_MAX_WORK_SIZE_PER_DIMENSION:568 return Info.write<ol_dimensions_t>(569 ol_dimensions_t{uint32_max, uint32_max, uint32_max});570 case OL_DEVICE_INFO_VENDOR_ID:571 return Info.write<uint32_t>(0);572 case OL_DEVICE_INFO_NUM_COMPUTE_UNITS:573 return Info.write<uint32_t>(1);574 case OL_DEVICE_INFO_SINGLE_FP_CONFIG:575 case OL_DEVICE_INFO_DOUBLE_FP_CONFIG:576 return Info.write<ol_device_fp_capability_flags_t>(577 OL_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT |578 OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST |579 OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO |580 OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF |581 OL_DEVICE_FP_CAPABILITY_FLAG_INF_NAN |582 OL_DEVICE_FP_CAPABILITY_FLAG_DENORM | OL_DEVICE_FP_CAPABILITY_FLAG_FMA);583 case OL_DEVICE_INFO_HALF_FP_CONFIG:584 return Info.write<ol_device_fp_capability_flags_t>(0);585 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_CHAR:586 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_SHORT:587 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_INT:588 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG:589 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_FLOAT:590 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE:591 return Info.write<uint32_t>(1);592 case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF:593 return Info.write<uint32_t>(0);594 case OL_DEVICE_INFO_MAX_CLOCK_FREQUENCY:595 case OL_DEVICE_INFO_MEMORY_CLOCK_RATE:596 case OL_DEVICE_INFO_ADDRESS_BITS:597 return Info.write<uint32_t>(std::numeric_limits<uintptr_t>::digits);598 case OL_DEVICE_INFO_MAX_MEM_ALLOC_SIZE:599 case OL_DEVICE_INFO_GLOBAL_MEM_SIZE:600 case OL_DEVICE_INFO_WORK_GROUP_LOCAL_MEM_SIZE:601 return Info.write<uint64_t>(0);602 default:603 return createOffloadError(ErrorCode::INVALID_ENUMERATION,604 "getDeviceInfo enum '%i' is invalid", PropName);605 }606 607 return Error::success();608}609 610Error olGetDeviceInfo_impl(ol_device_handle_t Device, ol_device_info_t PropName,611 size_t PropSize, void *PropValue) {612 if (Device == OffloadContext::get().HostDevice)613 return olGetDeviceInfoImplDetailHost(Device, PropName, PropSize, PropValue,614 nullptr);615 return olGetDeviceInfoImplDetail(Device, PropName, PropSize, PropValue,616 nullptr);617}618 619Error olGetDeviceInfoSize_impl(ol_device_handle_t Device,620 ol_device_info_t PropName, size_t *PropSizeRet) {621 if (Device == OffloadContext::get().HostDevice)622 return olGetDeviceInfoImplDetailHost(Device, PropName, 0, nullptr,623 PropSizeRet);624 return olGetDeviceInfoImplDetail(Device, PropName, 0, nullptr, PropSizeRet);625}626 627Error olIterateDevices_impl(ol_device_iterate_cb_t Callback, void *UserData) {628 for (auto &Platform : OffloadContext::get().Platforms) {629 for (auto &Device : Platform->Devices) {630 if (!Callback(Device.get(), UserData)) {631 return Error::success();632 }633 }634 }635 636 return Error::success();637}638 639TargetAllocTy convertOlToPluginAllocTy(ol_alloc_type_t Type) {640 switch (Type) {641 case OL_ALLOC_TYPE_DEVICE:642 return TARGET_ALLOC_DEVICE;643 case OL_ALLOC_TYPE_HOST:644 return TARGET_ALLOC_HOST;645 case OL_ALLOC_TYPE_MANAGED:646 default:647 return TARGET_ALLOC_SHARED;648 }649}650 651constexpr size_t MAX_ALLOC_TRIES = 50;652Error olMemAlloc_impl(ol_device_handle_t Device, ol_alloc_type_t Type,653 size_t Size, void **AllocationOut) {654 SmallVector<void *> Rejects;655 656 // Repeat the allocation up to a certain amount of times. If it happens to657 // already be allocated (e.g. by a device from another vendor) throw it away658 // and try again.659 for (size_t Count = 0; Count < MAX_ALLOC_TRIES; Count++) {660 auto NewAlloc = Device->Device->dataAlloc(Size, nullptr,661 convertOlToPluginAllocTy(Type));662 if (!NewAlloc)663 return NewAlloc.takeError();664 665 void *NewEnd = &static_cast<char *>(*NewAlloc)[Size];666 auto &AllocBases = OffloadContext::get().AllocBases;667 auto &AllocInfoMap = OffloadContext::get().AllocInfoMap;668 {669 std::lock_guard<std::mutex> Lock(OffloadContext::get().AllocInfoMapMutex);670 671 // Check that this memory region doesn't overlap another one672 // That is, the start of this allocation needs to be after another673 // allocation's end point, and the end of this allocation needs to be674 // before the next one's start.675 // `Gap` is the first alloc who ends after the new alloc's start point.676 auto Gap =677 std::lower_bound(AllocBases.begin(), AllocBases.end(), *NewAlloc,678 [&](const void *Iter, const void *Val) {679 return AllocInfoMap.at(Iter).End <= Val;680 });681 if (Gap == AllocBases.end() || NewEnd <= AllocInfoMap.at(*Gap).Start) {682 // Success, no conflict683 AllocInfoMap.insert_or_assign(684 *NewAlloc, AllocInfo{Device, Type, *NewAlloc, NewEnd});685 AllocBases.insert(686 std::lower_bound(AllocBases.begin(), AllocBases.end(), *NewAlloc),687 *NewAlloc);688 *AllocationOut = *NewAlloc;689 690 for (void *R : Rejects)691 if (auto Err =692 Device->Device->dataDelete(R, convertOlToPluginAllocTy(Type)))693 return Err;694 return Error::success();695 }696 697 // To avoid the next attempt allocating the same memory we just freed, we698 // hold onto it until we complete the allocation699 Rejects.push_back(*NewAlloc);700 }701 }702 703 // We've tried multiple times, and can't allocate a non-overlapping region.704 return createOffloadError(ErrorCode::BACKEND_FAILURE,705 "failed to allocate non-overlapping memory");706}707 708Error olMemFree_impl(void *Address) {709 ol_device_handle_t Device;710 ol_alloc_type_t Type;711 {712 std::lock_guard<std::mutex> Lock(OffloadContext::get().AllocInfoMapMutex);713 if (!OffloadContext::get().AllocInfoMap.contains(Address))714 return createOffloadError(ErrorCode::INVALID_ARGUMENT,715 "address is not a known allocation");716 717 auto AllocInfo = OffloadContext::get().AllocInfoMap.at(Address);718 Device = AllocInfo.Device;719 Type = AllocInfo.Type;720 OffloadContext::get().AllocInfoMap.erase(Address);721 722 auto &Bases = OffloadContext::get().AllocBases;723 Bases.erase(std::lower_bound(Bases.begin(), Bases.end(), Address));724 }725 726 if (auto Res =727 Device->Device->dataDelete(Address, convertOlToPluginAllocTy(Type)))728 return Res;729 730 return Error::success();731}732 733Error olGetMemInfoImplDetail(const void *Ptr, ol_mem_info_t PropName,734 size_t PropSize, void *PropValue,735 size_t *PropSizeRet) {736 InfoWriter Info(PropSize, PropValue, PropSizeRet);737 std::lock_guard<std::mutex> Lock(OffloadContext::get().AllocInfoMapMutex);738 739 auto &AllocBases = OffloadContext::get().AllocBases;740 auto &AllocInfoMap = OffloadContext::get().AllocInfoMap;741 const AllocInfo *Alloc = nullptr;742 if (AllocInfoMap.contains(Ptr)) {743 // Fast case, we have been given the base pointer directly744 Alloc = &AllocInfoMap.at(Ptr);745 } else {746 // Slower case, we need to look up the base pointer first747 // Find the first memory allocation whose end is after the target pointer,748 // and then check to see if it is in range749 auto Loc = std::lower_bound(AllocBases.begin(), AllocBases.end(), Ptr,750 [&](const void *Iter, const void *Val) {751 return AllocInfoMap.at(Iter).End <= Val;752 });753 if (Loc == AllocBases.end() || Ptr < AllocInfoMap.at(*Loc).Start)754 return Plugin::error(ErrorCode::NOT_FOUND,755 "allocated memory information not found");756 Alloc = &AllocInfoMap.at(*Loc);757 }758 759 switch (PropName) {760 case OL_MEM_INFO_DEVICE:761 return Info.write<ol_device_handle_t>(Alloc->Device);762 case OL_MEM_INFO_BASE:763 return Info.write<void *>(Alloc->Start);764 case OL_MEM_INFO_SIZE:765 return Info.write<size_t>(static_cast<char *>(Alloc->End) -766 static_cast<char *>(Alloc->Start));767 case OL_MEM_INFO_TYPE:768 return Info.write<ol_alloc_type_t>(Alloc->Type);769 default:770 return createOffloadError(ErrorCode::INVALID_ENUMERATION,771 "olGetMemInfo enum '%i' is invalid", PropName);772 }773 774 return Error::success();775}776 777Error olGetMemInfo_impl(const void *Ptr, ol_mem_info_t PropName,778 size_t PropSize, void *PropValue) {779 return olGetMemInfoImplDetail(Ptr, PropName, PropSize, PropValue, nullptr);780}781 782Error olGetMemInfoSize_impl(const void *Ptr, ol_mem_info_t PropName,783 size_t *PropSizeRet) {784 return olGetMemInfoImplDetail(Ptr, PropName, 0, nullptr, PropSizeRet);785}786 787Error olCreateQueue_impl(ol_device_handle_t Device, ol_queue_handle_t *Queue) {788 auto CreatedQueue = std::make_unique<ol_queue_impl_t>(nullptr, Device);789 790 auto OutstandingQueue = Device->getOutstandingQueue();791 if (OutstandingQueue) {792 // The queue is empty, but we still need to sync it to release any temporary793 // memory allocations or do other cleanup.794 if (auto Err =795 Device->Device->synchronize(OutstandingQueue, /*Release=*/false))796 return Err;797 CreatedQueue->AsyncInfo = OutstandingQueue;798 } else if (auto Err =799 Device->Device->initAsyncInfo(&(CreatedQueue->AsyncInfo))) {800 return Err;801 }802 803 *Queue = CreatedQueue.release();804 return Error::success();805}806 807Error olDestroyQueue_impl(ol_queue_handle_t Queue) {808 auto *Device = Queue->Device;809 // This is safe; as soon as olDestroyQueue is called it is not possible to add810 // any more work to the queue, so if it's finished now it will remain finished811 // forever.812 auto Res = Device->Device->hasPendingWork(Queue->AsyncInfo);813 if (!Res)814 return Res.takeError();815 816 if (!*Res) {817 // The queue is complete, so sync it and throw it back into the pool.818 if (auto Err = Device->Device->synchronize(Queue->AsyncInfo,819 /*Release=*/true))820 return Err;821 } else {822 // The queue still has outstanding work. Store it so we can check it later.823 std::lock_guard<std::mutex> Lock(Device->OutstandingQueuesMutex);824 Device->OutstandingQueues.push_back(Queue->AsyncInfo);825 }826 827 return olDestroy(Queue);828}829 830Error olSyncQueue_impl(ol_queue_handle_t Queue) {831 // Host plugin doesn't have a queue set so it's not safe to call synchronize832 // on it, but we have nothing to synchronize in that situation anyway.833 if (Queue->AsyncInfo->Queue) {834 // We don't need to release the queue and we would like the ability for835 // other offload threads to submit work concurrently, so pass "false" here836 // so we don't release the underlying queue object.837 if (auto Err = Queue->Device->Device->synchronize(Queue->AsyncInfo, false))838 return Err;839 }840 841 return Error::success();842}843 844Error olWaitEvents_impl(ol_queue_handle_t Queue, ol_event_handle_t *Events,845 size_t NumEvents) {846 auto *Device = Queue->Device->Device;847 848 for (size_t I = 0; I < NumEvents; I++) {849 auto *Event = Events[I];850 851 if (!Event)852 return Plugin::error(ErrorCode::INVALID_NULL_HANDLE,853 "olWaitEvents asked to wait on a NULL event");854 855 // Do nothing if the event is for this queue or the event is always complete856 if (Event->QueueId == Queue->Id || !Event->EventInfo)857 continue;858 859 if (auto Err = Device->waitEvent(Event->EventInfo, Queue->AsyncInfo))860 return Err;861 }862 863 return Error::success();864}865 866Error olGetQueueInfoImplDetail(ol_queue_handle_t Queue,867 ol_queue_info_t PropName, size_t PropSize,868 void *PropValue, size_t *PropSizeRet) {869 InfoWriter Info(PropSize, PropValue, PropSizeRet);870 871 switch (PropName) {872 case OL_QUEUE_INFO_DEVICE:873 return Info.write<ol_device_handle_t>(Queue->Device);874 case OL_QUEUE_INFO_EMPTY: {875 auto Pending = Queue->Device->Device->hasPendingWork(Queue->AsyncInfo);876 if (auto Err = Pending.takeError())877 return Err;878 return Info.write<bool>(!*Pending);879 }880 default:881 return createOffloadError(ErrorCode::INVALID_ENUMERATION,882 "olGetQueueInfo enum '%i' is invalid", PropName);883 }884 885 return Error::success();886}887 888Error olGetQueueInfo_impl(ol_queue_handle_t Queue, ol_queue_info_t PropName,889 size_t PropSize, void *PropValue) {890 return olGetQueueInfoImplDetail(Queue, PropName, PropSize, PropValue,891 nullptr);892}893 894Error olGetQueueInfoSize_impl(ol_queue_handle_t Queue, ol_queue_info_t PropName,895 size_t *PropSizeRet) {896 return olGetQueueInfoImplDetail(Queue, PropName, 0, nullptr, PropSizeRet);897}898 899Error olSyncEvent_impl(ol_event_handle_t Event) {900 // No event info means that this event was complete on creation901 if (!Event->EventInfo)902 return Plugin::success();903 904 if (auto Res = Event->Device->Device->syncEvent(Event->EventInfo))905 return Res;906 907 return Error::success();908}909 910Error olDestroyEvent_impl(ol_event_handle_t Event) {911 if (Event->EventInfo)912 if (auto Res = Event->Device->Device->destroyEvent(Event->EventInfo))913 return Res;914 915 return olDestroy(Event);916}917 918Error olGetEventInfoImplDetail(ol_event_handle_t Event,919 ol_event_info_t PropName, size_t PropSize,920 void *PropValue, size_t *PropSizeRet) {921 InfoWriter Info(PropSize, PropValue, PropSizeRet);922 auto Queue = Event->Queue;923 924 switch (PropName) {925 case OL_EVENT_INFO_QUEUE:926 return Info.write<ol_queue_handle_t>(Queue);927 case OL_EVENT_INFO_IS_COMPLETE: {928 // No event info means that this event was complete on creation929 if (!Event->EventInfo)930 return Info.write<bool>(true);931 932 auto Res = Queue->Device->Device->isEventComplete(Event->EventInfo,933 Queue->AsyncInfo);934 if (auto Err = Res.takeError())935 return Err;936 return Info.write<bool>(*Res);937 }938 default:939 return createOffloadError(ErrorCode::INVALID_ENUMERATION,940 "olGetEventInfo enum '%i' is invalid", PropName);941 }942 943 return Error::success();944}945 946Error olGetEventInfo_impl(ol_event_handle_t Event, ol_event_info_t PropName,947 size_t PropSize, void *PropValue) {948 949 return olGetEventInfoImplDetail(Event, PropName, PropSize, PropValue,950 nullptr);951}952 953Error olGetEventInfoSize_impl(ol_event_handle_t Event, ol_event_info_t PropName,954 size_t *PropSizeRet) {955 return olGetEventInfoImplDetail(Event, PropName, 0, nullptr, PropSizeRet);956}957 958Error olCreateEvent_impl(ol_queue_handle_t Queue, ol_event_handle_t *EventOut) {959 auto Pending = Queue->Device->Device->hasPendingWork(Queue->AsyncInfo);960 if (auto Err = Pending.takeError())961 return Err;962 963 *EventOut = new ol_event_impl_t(nullptr, Queue->Device, Queue);964 if (!*Pending)965 // Queue is empty, don't record an event and consider the event always966 // complete967 return Plugin::success();968 969 if (auto Res = Queue->Device->Device->createEvent(&(*EventOut)->EventInfo))970 return Res;971 972 if (auto Res = Queue->Device->Device->recordEvent((*EventOut)->EventInfo,973 Queue->AsyncInfo))974 return Res;975 976 return Plugin::success();977}978 979Error olMemcpy_impl(ol_queue_handle_t Queue, void *DstPtr,980 ol_device_handle_t DstDevice, const void *SrcPtr,981 ol_device_handle_t SrcDevice, size_t Size) {982 auto Host = OffloadContext::get().HostDevice;983 if (DstDevice == Host && SrcDevice == Host) {984 if (!Queue) {985 std::memcpy(DstPtr, SrcPtr, Size);986 return Error::success();987 } else {988 return createOffloadError(989 ErrorCode::INVALID_ARGUMENT,990 "ane of DstDevice and SrcDevice must be a non-host device if "991 "queue is specified");992 }993 }994 995 // If no queue is given the memcpy will be synchronous996 auto QueueImpl = Queue ? Queue->AsyncInfo : nullptr;997 998 if (DstDevice == Host) {999 if (auto Res =1000 SrcDevice->Device->dataRetrieve(DstPtr, SrcPtr, Size, QueueImpl))1001 return Res;1002 } else if (SrcDevice == Host) {1003 if (auto Res =1004 DstDevice->Device->dataSubmit(DstPtr, SrcPtr, Size, QueueImpl))1005 return Res;1006 } else {1007 if (auto Res = SrcDevice->Device->dataExchange(SrcPtr, *DstDevice->Device,1008 DstPtr, Size, QueueImpl))1009 return Res;1010 }1011 1012 return Error::success();1013}1014 1015Error olMemFill_impl(ol_queue_handle_t Queue, void *Ptr, size_t PatternSize,1016 const void *PatternPtr, size_t FillSize) {1017 return Queue->Device->Device->dataFill(Ptr, PatternPtr, PatternSize, FillSize,1018 Queue->AsyncInfo);1019}1020 1021Error olCreateProgram_impl(ol_device_handle_t Device, const void *ProgData,1022 size_t ProgDataSize, ol_program_handle_t *Program) {1023 StringRef Buffer(reinterpret_cast<const char *>(ProgData), ProgDataSize);1024 Expected<plugin::DeviceImageTy *> Res =1025 Device->Device->loadBinary(Device->Device->Plugin, Buffer);1026 if (!Res)1027 return Res.takeError();1028 assert(*Res && "loadBinary returned nullptr");1029 1030 *Program = new ol_program_impl_t(*Res, (*Res)->getMemoryBuffer());1031 return Error::success();1032}1033 1034Error olIsValidBinary_impl(ol_device_handle_t Device, const void *ProgData,1035 size_t ProgDataSize, bool *IsValid) {1036 StringRef Buffer(reinterpret_cast<const char *>(ProgData), ProgDataSize);1037 *IsValid = Device->Device ? Device->Device->Plugin.isDeviceCompatible(1038 Device->Device->getDeviceId(), Buffer)1039 : false;1040 return Error::success();1041}1042 1043Error olDestroyProgram_impl(ol_program_handle_t Program) {1044 auto &Device = Program->Image->getDevice();1045 if (auto Err = Device.unloadBinary(Program->Image))1046 return Err;1047 1048 auto &LoadedImages = Device.LoadedImages;1049 LoadedImages.erase(1050 std::find(LoadedImages.begin(), LoadedImages.end(), Program->Image));1051 1052 return olDestroy(Program);1053}1054 1055Error olCalculateOptimalOccupancy_impl(ol_device_handle_t Device,1056 ol_symbol_handle_t Kernel,1057 size_t DynamicMemSize,1058 size_t *GroupSize) {1059 if (Kernel->Kind != OL_SYMBOL_KIND_KERNEL)1060 return createOffloadError(ErrorCode::SYMBOL_KIND,1061 "provided symbol is not a kernel");1062 auto *KernelImpl = std::get<GenericKernelTy *>(Kernel->PluginImpl);1063 1064 auto Res = KernelImpl->maxGroupSize(*Device->Device, DynamicMemSize);1065 if (auto Err = Res.takeError())1066 return Err;1067 1068 *GroupSize = *Res;1069 1070 return Error::success();1071}1072 1073Error olLaunchKernel_impl(ol_queue_handle_t Queue, ol_device_handle_t Device,1074 ol_symbol_handle_t Kernel, const void *ArgumentsData,1075 size_t ArgumentsSize,1076 const ol_kernel_launch_size_args_t *LaunchSizeArgs) {1077 auto *DeviceImpl = Device->Device;1078 if (Queue && Device != Queue->Device) {1079 return createOffloadError(1080 ErrorCode::INVALID_DEVICE,1081 "device specified does not match the device of the given queue");1082 }1083 1084 if (Kernel->Kind != OL_SYMBOL_KIND_KERNEL)1085 return createOffloadError(ErrorCode::SYMBOL_KIND,1086 "provided symbol is not a kernel");1087 1088 auto *QueueImpl = Queue ? Queue->AsyncInfo : nullptr;1089 AsyncInfoWrapperTy AsyncInfoWrapper(*DeviceImpl, QueueImpl);1090 KernelArgsTy LaunchArgs{};1091 LaunchArgs.NumTeams[0] = LaunchSizeArgs->NumGroups.x;1092 LaunchArgs.NumTeams[1] = LaunchSizeArgs->NumGroups.y;1093 LaunchArgs.NumTeams[2] = LaunchSizeArgs->NumGroups.z;1094 LaunchArgs.ThreadLimit[0] = LaunchSizeArgs->GroupSize.x;1095 LaunchArgs.ThreadLimit[1] = LaunchSizeArgs->GroupSize.y;1096 LaunchArgs.ThreadLimit[2] = LaunchSizeArgs->GroupSize.z;1097 LaunchArgs.DynCGroupMem = LaunchSizeArgs->DynSharedMemory;1098 1099 KernelLaunchParamsTy Params;1100 Params.Data = const_cast<void *>(ArgumentsData);1101 Params.Size = ArgumentsSize;1102 LaunchArgs.ArgPtrs = reinterpret_cast<void **>(&Params);1103 // Don't do anything with pointer indirection; use arg data as-is1104 LaunchArgs.Flags.IsCUDA = true;1105 1106 auto *KernelImpl = std::get<GenericKernelTy *>(Kernel->PluginImpl);1107 auto Err = KernelImpl->launch(*DeviceImpl, LaunchArgs.ArgPtrs, nullptr,1108 LaunchArgs, AsyncInfoWrapper);1109 1110 AsyncInfoWrapper.finalize(Err);1111 if (Err)1112 return Err;1113 1114 return Error::success();1115}1116 1117Error olGetSymbol_impl(ol_program_handle_t Program, const char *Name,1118 ol_symbol_kind_t Kind, ol_symbol_handle_t *Symbol) {1119 auto &Device = Program->Image->getDevice();1120 1121 std::lock_guard<std::mutex> Lock(Program->SymbolListMutex);1122 1123 switch (Kind) {1124 case OL_SYMBOL_KIND_KERNEL: {1125 auto &Kernel = Program->KernelSymbols[Name];1126 if (!Kernel) {1127 auto KernelImpl = Device.constructKernel(Name);1128 if (!KernelImpl)1129 return KernelImpl.takeError();1130 1131 if (auto Err = KernelImpl->init(Device, *Program->Image))1132 return Err;1133 1134 Kernel = std::make_unique<ol_symbol_impl_t>(KernelImpl->getName(),1135 &*KernelImpl);1136 }1137 1138 *Symbol = Kernel.get();1139 return Error::success();1140 }1141 case OL_SYMBOL_KIND_GLOBAL_VARIABLE: {1142 auto &Global = Program->GlobalSymbols[Name];1143 if (!Global) {1144 GlobalTy GlobalObj{Name};1145 if (auto Res =1146 Device.Plugin.getGlobalHandler().getGlobalMetadataFromDevice(1147 Device, *Program->Image, GlobalObj))1148 return Res;1149 1150 Global = std::make_unique<ol_symbol_impl_t>(GlobalObj.getName().c_str(),1151 std::move(GlobalObj));1152 }1153 1154 *Symbol = Global.get();1155 return Error::success();1156 }1157 default:1158 return createOffloadError(ErrorCode::INVALID_ENUMERATION,1159 "getSymbol kind enum '%i' is invalid", Kind);1160 }1161}1162 1163Error olGetSymbolInfoImplDetail(ol_symbol_handle_t Symbol,1164 ol_symbol_info_t PropName, size_t PropSize,1165 void *PropValue, size_t *PropSizeRet) {1166 InfoWriter Info(PropSize, PropValue, PropSizeRet);1167 1168 auto CheckKind = [&](ol_symbol_kind_t Required) {1169 if (Symbol->Kind != Required) {1170 std::string ErrBuffer;1171 raw_string_ostream(ErrBuffer)1172 << PropName << ": Expected a symbol of Kind " << Required1173 << " but given a symbol of Kind " << Symbol->Kind;1174 return Plugin::error(ErrorCode::SYMBOL_KIND, ErrBuffer.c_str());1175 }1176 return Plugin::success();1177 };1178 1179 switch (PropName) {1180 case OL_SYMBOL_INFO_KIND:1181 return Info.write<ol_symbol_kind_t>(Symbol->Kind);1182 case OL_SYMBOL_INFO_GLOBAL_VARIABLE_ADDRESS:1183 if (auto Err = CheckKind(OL_SYMBOL_KIND_GLOBAL_VARIABLE))1184 return Err;1185 return Info.write<void *>(std::get<GlobalTy>(Symbol->PluginImpl).getPtr());1186 case OL_SYMBOL_INFO_GLOBAL_VARIABLE_SIZE:1187 if (auto Err = CheckKind(OL_SYMBOL_KIND_GLOBAL_VARIABLE))1188 return Err;1189 return Info.write<size_t>(std::get<GlobalTy>(Symbol->PluginImpl).getSize());1190 default:1191 return createOffloadError(ErrorCode::INVALID_ENUMERATION,1192 "olGetSymbolInfo enum '%i' is invalid", PropName);1193 }1194 1195 return Error::success();1196}1197 1198Error olGetSymbolInfo_impl(ol_symbol_handle_t Symbol, ol_symbol_info_t PropName,1199 size_t PropSize, void *PropValue) {1200 1201 return olGetSymbolInfoImplDetail(Symbol, PropName, PropSize, PropValue,1202 nullptr);1203}1204 1205Error olGetSymbolInfoSize_impl(ol_symbol_handle_t Symbol,1206 ol_symbol_info_t PropName, size_t *PropSizeRet) {1207 return olGetSymbolInfoImplDetail(Symbol, PropName, 0, nullptr, PropSizeRet);1208}1209 1210Error olLaunchHostFunction_impl(ol_queue_handle_t Queue,1211 ol_host_function_cb_t Callback,1212 void *UserData) {1213 return Queue->Device->Device->enqueueHostCall(Callback, UserData,1214 Queue->AsyncInfo);1215}1216 1217} // namespace offload1218} // namespace llvm1219