569 lines · cpp
1//===-- PluginManager.cpp - Plugin loading and communication API ---------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Functionality for handling plugins.10//11//===----------------------------------------------------------------------===//12 13#include "PluginManager.h"14#include "OffloadPolicy.h"15#include "Shared/Debug.h"16#include "Shared/Profile.h"17#include "device.h"18 19#include "llvm/Support/Error.h"20#include "llvm/Support/ErrorHandling.h"21#include <memory>22 23using namespace llvm;24using namespace llvm::sys;25 26PluginManager *PM = nullptr;27 28// Every plugin exports this method to create an instance of the plugin type.29#define PLUGIN_TARGET(Name) extern "C" GenericPluginTy *createPlugin_##Name();30#include "Shared/Targets.def"31 32void PluginManager::init() {33 TIMESCOPE();34 if (OffloadPolicy::isOffloadDisabled()) {35 DP("Offload is disabled. Skipping plugin initialization\n");36 return;37 }38 39 ODBG("Init") << "Loading RTLs";40 41 // Attempt to create an instance of each supported plugin.42#define PLUGIN_TARGET(Name) \43 do { \44 Plugins.emplace_back( \45 std::unique_ptr<GenericPluginTy>(createPlugin_##Name())); \46 } while (false);47#include "Shared/Targets.def"48 49 DP("RTLs loaded!\n");50}51 52void PluginManager::deinit() {53 TIMESCOPE();54 DP("Unloading RTLs...\n");55 56 for (auto &Plugin : Plugins) {57 if (!Plugin->is_initialized())58 continue;59 60 if (auto Err = Plugin->deinit()) {61 [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));62 DP("Failed to deinit plugin: %s\n", InfoMsg.c_str());63 }64 Plugin.release();65 }66 67 DP("RTLs unloaded!\n");68}69 70bool PluginManager::initializePlugin(GenericPluginTy &Plugin) {71 if (Plugin.is_initialized())72 return true;73 74 if (auto Err = Plugin.init()) {75 [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));76 DP("Failed to init plugin: %s\n", InfoMsg.c_str());77 return false;78 }79 80 DP("Registered plugin %s with %d visible device(s)\n", Plugin.getName(),81 Plugin.number_of_devices());82 return true;83}84 85bool PluginManager::initializeDevice(GenericPluginTy &Plugin,86 int32_t DeviceId) {87 if (Plugin.is_device_initialized(DeviceId)) {88 auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();89 (*ExclusiveDevicesAccessor)[PM->DeviceIds[std::make_pair(&Plugin,90 DeviceId)]]91 ->setHasPendingImages(true);92 return true;93 }94 95 // Initialize the device information for the RTL we are about to use.96 auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();97 98 int32_t UserId = ExclusiveDevicesAccessor->size();99 100 // Set the device identifier offset in the plugin.101#ifdef OMPT_SUPPORT102 Plugin.set_device_identifier(UserId, DeviceId);103#endif104 105 auto Device = std::make_unique<DeviceTy>(&Plugin, UserId, DeviceId);106 if (auto Err = Device->init()) {107 [[maybe_unused]] std::string InfoMsg = toString(std::move(Err));108 DP("Failed to init device %d: %s\n", DeviceId, InfoMsg.c_str());109 return false;110 }111 112 ExclusiveDevicesAccessor->push_back(std::move(Device));113 114 // We need to map between the plugin's device identifier and the one115 // that OpenMP will use.116 PM->DeviceIds[std::make_pair(&Plugin, DeviceId)] = UserId;117 118 return true;119}120 121void PluginManager::initializeAllDevices() {122 for (auto &Plugin : plugins()) {123 if (!initializePlugin(Plugin))124 continue;125 126 for (int32_t DeviceId = 0; DeviceId < Plugin.number_of_devices();127 ++DeviceId) {128 initializeDevice(Plugin, DeviceId);129 }130 }131 // After all plugins are initialized, register atExit cleanup handlers132 std::atexit([]() {133 // Interop cleanup should be done before the plugins are deinitialized as134 // the backend libraries may be already unloaded.135 if (PM)136 PM->InteropTbl.clear();137 });138}139 140// Returns a pointer to the binary descriptor, upgrading from a legacy format if141// necessary.142__tgt_bin_desc *PluginManager::upgradeLegacyEntries(__tgt_bin_desc *Desc) {143 struct LegacyEntryTy {144 void *Address;145 char *SymbolName;146 size_t Size;147 int32_t Flags;148 int32_t Data;149 };150 151 if (UpgradedDescriptors.contains(Desc))152 return &UpgradedDescriptors[Desc];153 154 if (Desc->HostEntriesBegin == Desc->HostEntriesEnd ||155 Desc->HostEntriesBegin->Reserved == 0)156 return Desc;157 158 // The new format mandates that each entry starts with eight bytes of zeroes.159 // This allows us to detect the old format as this is a null pointer.160 llvm::SmallVector<llvm::offloading::EntryTy, 0> &NewEntries =161 LegacyEntries.emplace_back();162 for (LegacyEntryTy &Entry : llvm::make_range(163 reinterpret_cast<LegacyEntryTy *>(Desc->HostEntriesBegin),164 reinterpret_cast<LegacyEntryTy *>(Desc->HostEntriesEnd))) {165 llvm::offloading::EntryTy &NewEntry = NewEntries.emplace_back();166 167 NewEntry.Address = Entry.Address;168 NewEntry.Flags = Entry.Flags;169 NewEntry.Data = Entry.Data;170 NewEntry.Size = Entry.Size;171 NewEntry.SymbolName = Entry.SymbolName;172 NewEntry.Kind = object::OffloadKind::OFK_OpenMP;173 }174 175 // Create a new image struct so we can update the entries list.176 llvm::SmallVector<__tgt_device_image, 0> &NewImages =177 LegacyImages.emplace_back();178 for (int32_t Image = 0; Image < Desc->NumDeviceImages; ++Image)179 NewImages.emplace_back(180 __tgt_device_image{Desc->DeviceImages[Image].ImageStart,181 Desc->DeviceImages[Image].ImageEnd,182 NewEntries.begin(), NewEntries.end()});183 184 // Create the new binary descriptor containing the newly created memory.185 __tgt_bin_desc &NewDesc = UpgradedDescriptors[Desc];186 NewDesc.DeviceImages = NewImages.begin();187 NewDesc.NumDeviceImages = Desc->NumDeviceImages;188 NewDesc.HostEntriesBegin = NewEntries.begin();189 NewDesc.HostEntriesEnd = NewEntries.end();190 191 return &NewDesc;192}193 194void PluginManager::registerLib(__tgt_bin_desc *Desc) {195 PM->RTLsMtx.lock();196 197 // Upgrade the entries from the legacy implementation if necessary.198 Desc = upgradeLegacyEntries(Desc);199 200 // Add in all the OpenMP requirements associated with this binary.201 for (llvm::offloading::EntryTy &Entry :202 llvm::make_range(Desc->HostEntriesBegin, Desc->HostEntriesEnd))203 if (Entry.Kind == object::OffloadKind::OFK_OpenMP &&204 Entry.Flags == OMP_REGISTER_REQUIRES)205 PM->addRequirements(Entry.Data);206 207 // Extract the executable image and extra information if available.208 for (int32_t i = 0; i < Desc->NumDeviceImages; ++i)209 PM->addDeviceImage(*Desc, Desc->DeviceImages[i]);210 211 // Register the images with the RTLs that understand them, if any.212 llvm::DenseMap<GenericPluginTy *, llvm::DenseSet<int32_t>> UsedDevices;213 for (int32_t i = 0; i < Desc->NumDeviceImages; ++i) {214 // Obtain the image and information that was previously extracted.215 __tgt_device_image *Img = &Desc->DeviceImages[i];216 217 GenericPluginTy *FoundRTL = nullptr;218 219 // Scan the RTLs that have associated images until we find one that supports220 // the current image.221 for (auto &R : plugins()) {222 StringRef Buffer(reinterpret_cast<const char *>(Img->ImageStart),223 utils::getPtrDiff(Img->ImageEnd, Img->ImageStart));224 225 if (!R.isPluginCompatible(Buffer))226 continue;227 228 if (!initializePlugin(R))229 continue;230 231 if (!R.number_of_devices()) {232 DP("Skipping plugin %s with no visible devices\n", R.getName());233 continue;234 }235 236 for (int32_t DeviceId = 0; DeviceId < R.number_of_devices(); ++DeviceId) {237 // We only want a single matching image to be registered for each binary238 // descriptor. This prevents multiple of the same image from being239 // registered for the same device in the case that they are mutually240 // compatible, such as sm_80 and sm_89.241 if (UsedDevices[&R].contains(DeviceId)) {242 DP("Image " DPxMOD243 " is a duplicate, not loaded on RTL %s device %d!\n",244 DPxPTR(Img->ImageStart), R.getName(), DeviceId);245 continue;246 }247 248 if (!R.isDeviceCompatible(DeviceId, Buffer))249 continue;250 251 DP("Image " DPxMOD " is compatible with RTL %s device %d!\n",252 DPxPTR(Img->ImageStart), R.getName(), DeviceId);253 254 if (!initializeDevice(R, DeviceId))255 continue;256 257 // Initialize (if necessary) translation table for this library.258 PM->TrlTblMtx.lock();259 if (!PM->HostEntriesBeginToTransTable.count(Desc->HostEntriesBegin)) {260 PM->HostEntriesBeginRegistrationOrder.push_back(261 Desc->HostEntriesBegin);262 TranslationTable &TT =263 (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin];264 TT.HostTable.EntriesBegin = Desc->HostEntriesBegin;265 TT.HostTable.EntriesEnd = Desc->HostEntriesEnd;266 }267 268 // Retrieve translation table for this library.269 TranslationTable &TT =270 (PM->HostEntriesBeginToTransTable)[Desc->HostEntriesBegin];271 272 DP("Registering image " DPxMOD " with RTL %s!\n",273 DPxPTR(Img->ImageStart), R.getName());274 275 auto UserId = PM->DeviceIds[std::make_pair(&R, DeviceId)];276 if (TT.TargetsTable.size() < static_cast<size_t>(UserId + 1)) {277 TT.DeviceTables.resize(UserId + 1, {});278 TT.TargetsImages.resize(UserId + 1, nullptr);279 TT.TargetsEntries.resize(UserId + 1, {});280 TT.TargetsTable.resize(UserId + 1, nullptr);281 }282 283 // Register the image for this target type and invalidate the table.284 TT.TargetsImages[UserId] = Img;285 TT.TargetsTable[UserId] = nullptr;286 287 UsedDevices[&R].insert(DeviceId);288 PM->UsedImages.insert(Img);289 FoundRTL = &R;290 291 PM->TrlTblMtx.unlock();292 }293 }294 if (!FoundRTL)295 DP("No RTL found for image " DPxMOD "!\n", DPxPTR(Img->ImageStart));296 }297 PM->RTLsMtx.unlock();298 299 bool UseAutoZeroCopy = false;300 301 auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();302 // APUs are homogeneous set of GPUs. Check the first device for303 // configuring Auto Zero-Copy.304 if (ExclusiveDevicesAccessor->size() > 0) {305 auto &Device = *(*ExclusiveDevicesAccessor)[0];306 UseAutoZeroCopy = Device.useAutoZeroCopy();307 }308 309 if (UseAutoZeroCopy)310 addRequirements(OMPX_REQ_AUTO_ZERO_COPY);311 312 DP("Done registering entries!\n");313}314 315// Temporary forward declaration, old style CTor/DTor handling is going away.316int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,317 KernelArgsTy &KernelArgs, AsyncInfoTy &AsyncInfo);318 319void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {320 DP("Unloading target library!\n");321 322 Desc = upgradeLegacyEntries(Desc);323 324 PM->RTLsMtx.lock();325 // Find which RTL understands each image, if any.326 for (DeviceImageTy &DI : PM->deviceImages()) {327 // Obtain the image and information that was previously extracted.328 __tgt_device_image *Img = &DI.getExecutableImage();329 330 GenericPluginTy *FoundRTL = NULL;331 332 // Scan the RTLs that have associated images until we find one that supports333 // the current image. We only need to scan RTLs that are already being used.334 for (auto &R : plugins()) {335 if (R.is_initialized())336 continue;337 338 // Ensure that we do not use any unused images associated with this RTL.339 if (!UsedImages.contains(Img))340 continue;341 342 FoundRTL = &R;343 344 DP("Unregistered image " DPxMOD " from RTL\n", DPxPTR(Img->ImageStart));345 346 break;347 }348 349 // if no RTL was found proceed to unregister the next image350 if (!FoundRTL) {351 DP("No RTLs in use support the image " DPxMOD "!\n",352 DPxPTR(Img->ImageStart));353 }354 }355 PM->RTLsMtx.unlock();356 DP("Done unregistering images!\n");357 358 // Remove entries from PM->HostPtrToTableMap359 PM->TblMapMtx.lock();360 for (llvm::offloading::EntryTy *Cur = Desc->HostEntriesBegin;361 Cur < Desc->HostEntriesEnd; ++Cur) {362 if (Cur->Kind == object::OffloadKind::OFK_OpenMP)363 PM->HostPtrToTableMap.erase(Cur->Address);364 }365 366 // Remove translation table for this descriptor.367 auto TransTable =368 PM->HostEntriesBeginToTransTable.find(Desc->HostEntriesBegin);369 if (TransTable != PM->HostEntriesBeginToTransTable.end()) {370 DP("Removing translation table for descriptor " DPxMOD "\n",371 DPxPTR(Desc->HostEntriesBegin));372 PM->HostEntriesBeginToTransTable.erase(TransTable);373 } else {374 DP("Translation table for descriptor " DPxMOD " cannot be found, probably "375 "it has been already removed.\n",376 DPxPTR(Desc->HostEntriesBegin));377 }378 379 PM->TblMapMtx.unlock();380 381 DP("Done unregistering library!\n");382}383 384/// Map global data and execute pending ctors385static int loadImagesOntoDevice(DeviceTy &Device) {386 /*387 * Map global data388 */389 int32_t DeviceId = Device.DeviceID;390 int Rc = OFFLOAD_SUCCESS;391 {392 std::lock_guard<decltype(PM->TrlTblMtx)> LG(PM->TrlTblMtx);393 for (auto *HostEntriesBegin : PM->HostEntriesBeginRegistrationOrder) {394 TranslationTable *TransTable =395 &PM->HostEntriesBeginToTransTable[HostEntriesBegin];396 DP("Trans table %p : %p\n", TransTable->HostTable.EntriesBegin,397 TransTable->HostTable.EntriesEnd);398 if (TransTable->HostTable.EntriesBegin ==399 TransTable->HostTable.EntriesEnd) {400 // No host entry so no need to proceed401 continue;402 }403 404 if (TransTable->TargetsTable[DeviceId] != 0) {405 // Library entries have already been processed406 continue;407 }408 409 // 1) get image.410 assert(TransTable->TargetsImages.size() > (size_t)DeviceId &&411 "Not expecting a device ID outside the table's bounds!");412 __tgt_device_image *Img = TransTable->TargetsImages[DeviceId];413 if (!Img) {414 REPORT("No image loaded for device id %d.\n", DeviceId);415 Rc = OFFLOAD_FAIL;416 break;417 }418 419 // 2) Load the image onto the given device.420 auto BinaryOrErr = Device.loadBinary(Img);421 if (llvm::Error Err = BinaryOrErr.takeError()) {422 REPORT("Failed to load image %s\n",423 llvm::toString(std::move(Err)).c_str());424 Rc = OFFLOAD_FAIL;425 break;426 }427 428 // 3) Create the translation table.429 llvm::SmallVector<llvm::offloading::EntryTy> &DeviceEntries =430 TransTable->TargetsEntries[DeviceId];431 for (llvm::offloading::EntryTy &Entry :432 llvm::make_range(Img->EntriesBegin, Img->EntriesEnd)) {433 if (Entry.Kind != object::OffloadKind::OFK_OpenMP)434 continue;435 436 __tgt_device_binary &Binary = *BinaryOrErr;437 438 llvm::offloading::EntryTy DeviceEntry = Entry;439 if (Entry.Size) {440 if (!(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT_VTABLE))441 if (Device.RTL->get_global(Binary, Entry.Size, Entry.SymbolName,442 &DeviceEntry.Address) != OFFLOAD_SUCCESS)443 REPORT("Failed to load symbol %s\n", Entry.SymbolName);444 445 // If unified memory is active, the corresponding global is a device446 // reference to the host global. We need to initialize the pointer on447 // the device to point to the memory on the host.448 if (!(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT_VTABLE) &&449 !(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT) &&450 ((PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY) ||451 (PM->getRequirements() & OMPX_REQ_AUTO_ZERO_COPY)))452 if (Device.RTL->data_submit(DeviceId, DeviceEntry.Address,453 Entry.Address,454 Entry.Size) != OFFLOAD_SUCCESS)455 REPORT("Failed to write symbol for USM %s\n", Entry.SymbolName);456 } else if (Entry.Address) {457 if (Device.RTL->get_function(Binary, Entry.SymbolName,458 &DeviceEntry.Address) != OFFLOAD_SUCCESS)459 REPORT("Failed to load kernel %s\n", Entry.SymbolName);460 }461 DP("Entry point " DPxMOD " maps to%s %s (" DPxMOD ")\n",462 DPxPTR(Entry.Address), (Entry.Size) ? " global" : "",463 Entry.SymbolName, DPxPTR(DeviceEntry.Address));464 465 DeviceEntries.emplace_back(DeviceEntry);466 }467 468 // Set the storage for the table and get a pointer to it.469 __tgt_target_table DeviceTable{&DeviceEntries[0],470 &DeviceEntries[0] + DeviceEntries.size()};471 TransTable->DeviceTables[DeviceId] = DeviceTable;472 __tgt_target_table *TargetTable = TransTable->TargetsTable[DeviceId] =473 &TransTable->DeviceTables[DeviceId];474 475 MappingInfoTy::HDTTMapAccessorTy HDTTMap =476 Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();477 478 __tgt_target_table *HostTable = &TransTable->HostTable;479 for (llvm::offloading::EntryTy *480 CurrDeviceEntry = TargetTable->EntriesBegin,481 *CurrHostEntry = HostTable->EntriesBegin,482 *EntryDeviceEnd = TargetTable->EntriesEnd;483 CurrDeviceEntry != EntryDeviceEnd;484 CurrDeviceEntry++, CurrHostEntry++) {485 if (CurrDeviceEntry->Size == 0 ||486 CurrDeviceEntry->Kind != object::OffloadKind::OFK_OpenMP)487 continue;488 489 assert(CurrDeviceEntry->Size == CurrHostEntry->Size &&490 "data size mismatch");491 492 // Fortran may use multiple weak declarations for the same symbol,493 // therefore we must allow for multiple weak symbols to be loaded from494 // the fat binary. Treat these mappings as any other "regular"495 // mapping. Add entry to map.496 if (Device.getMappingInfo().getTgtPtrBegin(497 HDTTMap, CurrHostEntry->Address, CurrHostEntry->Size))498 continue;499 500 void *CurrDeviceEntryAddr = CurrDeviceEntry->Address;501 502 // For indirect mapping, follow the indirection and map the actual503 // target.504 if (CurrDeviceEntry->Flags & OMP_DECLARE_TARGET_INDIRECT) {505 AsyncInfoTy AsyncInfo(Device);506 void *DevPtr;507 Device.retrieveData(&DevPtr, CurrDeviceEntryAddr, sizeof(void *),508 AsyncInfo, /*Entry=*/nullptr, &HDTTMap);509 if (AsyncInfo.synchronize() != OFFLOAD_SUCCESS)510 return OFFLOAD_FAIL;511 CurrDeviceEntryAddr = DevPtr;512 }513 514 DP("Add mapping from host " DPxMOD " to device " DPxMOD " with size %zu"515 ", name \"%s\"\n",516 DPxPTR(CurrHostEntry->Address), DPxPTR(CurrDeviceEntry->Address),517 CurrDeviceEntry->Size, CurrDeviceEntry->SymbolName);518 HDTTMap->emplace(new HostDataToTargetTy(519 (uintptr_t)CurrHostEntry->Address /*HstPtrBase*/,520 (uintptr_t)CurrHostEntry->Address /*HstPtrBegin*/,521 (uintptr_t)CurrHostEntry->Address +522 CurrHostEntry->Size /*HstPtrEnd*/,523 (uintptr_t)CurrDeviceEntryAddr /*TgtAllocBegin*/,524 (uintptr_t)CurrDeviceEntryAddr /*TgtPtrBegin*/,525 false /*UseHoldRefCount*/, CurrHostEntry->SymbolName,526 true /*IsRefCountINF*/));527 528 // Notify about the new mapping.529 if (Device.notifyDataMapped(CurrHostEntry->Address,530 CurrHostEntry->Size))531 return OFFLOAD_FAIL;532 }533 }534 Device.setHasPendingImages(false);535 }536 537 if (Rc != OFFLOAD_SUCCESS)538 return Rc;539 540 static Int32Envar DumpOffloadEntries =541 Int32Envar("OMPTARGET_DUMP_OFFLOAD_ENTRIES", -1);542 if (DumpOffloadEntries.get() == DeviceId)543 Device.dumpOffloadEntries();544 545 return OFFLOAD_SUCCESS;546}547 548Expected<DeviceTy &> PluginManager::getDevice(uint32_t DeviceNo) {549 DeviceTy *DevicePtr;550 {551 auto ExclusiveDevicesAccessor = getExclusiveDevicesAccessor();552 if (DeviceNo >= ExclusiveDevicesAccessor->size())553 return error::createOffloadError(554 error::ErrorCode::INVALID_VALUE,555 "device number '%i' out of range, only %i devices available",556 DeviceNo, ExclusiveDevicesAccessor->size());557 558 DevicePtr = &*(*ExclusiveDevicesAccessor)[DeviceNo];559 }560 561 // Check whether global data has been mapped for this device562 if (DevicePtr->hasPendingImages())563 if (loadImagesOntoDevice(*DevicePtr) != OFFLOAD_SUCCESS)564 return error::createOffloadError(error::ErrorCode::BACKEND_FAILURE,565 "failed to load images on device '%i'",566 DeviceNo);567 return *DevicePtr;568}569