411 lines · cpp
1//===--------- device.cpp - Target independent OpenMP target RTL ----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Functionality for managing devices that are handled by RTL plugins.10//11//===----------------------------------------------------------------------===//12 13#include "device.h"14#include "OffloadEntry.h"15#include "OpenMP/Mapping.h"16#include "OpenMP/OMPT/Callback.h"17#include "OpenMP/OMPT/Interface.h"18#include "PluginManager.h"19#include "Shared/APITypes.h"20#include "Shared/Debug.h"21#include "omptarget.h"22#include "private.h"23#include "rtl.h"24 25#include "Shared/EnvironmentVar.h"26#include "llvm/Support/Error.h"27 28#include <cassert>29#include <climits>30#include <cstdint>31#include <cstdio>32#include <mutex>33#include <string>34#include <thread>35 36#ifdef OMPT_SUPPORT37using namespace llvm::omp::target::ompt;38#endif39 40using namespace llvm::omp::target::plugin;41 42int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,43 AsyncInfoTy &AsyncInfo) const {44 // First, check if the user disabled atomic map transfer/malloc/dealloc.45 if (!MappingConfig::get().UseEventsForAtomicTransfers)46 return OFFLOAD_SUCCESS;47 48 void *Event = getEvent();49 bool NeedNewEvent = Event == nullptr;50 if (NeedNewEvent && Device.createEvent(&Event) != OFFLOAD_SUCCESS) {51 REPORT("Failed to create event\n");52 return OFFLOAD_FAIL;53 }54 55 // We cannot assume the event should not be nullptr because we don't56 // know if the target support event. But if a target doesn't,57 // recordEvent should always return success.58 if (Device.recordEvent(Event, AsyncInfo) != OFFLOAD_SUCCESS) {59 REPORT("Failed to set dependence on event " DPxMOD "\n", DPxPTR(Event));60 return OFFLOAD_FAIL;61 }62 63 if (NeedNewEvent)64 setEvent(Event);65 66 return OFFLOAD_SUCCESS;67}68 69DeviceTy::DeviceTy(GenericPluginTy *RTL, int32_t DeviceID, int32_t RTLDeviceID)70 : DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID),71 MappingInfo(*this) {}72 73DeviceTy::~DeviceTy() {74 if (DeviceID == -1 || !(getInfoLevel() & OMP_INFOTYPE_DUMP_TABLE))75 return;76 77 ident_t Loc = {0, 0, 0, 0, ";libomptarget;libomptarget;0;0;;"};78 dumpTargetPointerMappings(&Loc, *this);79}80 81llvm::Error DeviceTy::init() {82 int32_t Ret = RTL->init_device(RTLDeviceID);83 if (Ret != OFFLOAD_SUCCESS)84 return error::createOffloadError(error::ErrorCode::BACKEND_FAILURE,85 "failed to initialize device %d\n",86 DeviceID);87 88 // Enables recording kernels if set.89 BoolEnvar OMPX_RecordKernel("LIBOMPTARGET_RECORD", false);90 if (OMPX_RecordKernel) {91 // Enables saving the device memory kernel output post execution if set.92 BoolEnvar OMPX_ReplaySaveOutput("LIBOMPTARGET_RR_SAVE_OUTPUT", false);93 94 uint64_t ReqPtrArgOffset;95 RTL->initialize_record_replay(RTLDeviceID, 0, nullptr, true,96 OMPX_ReplaySaveOutput, ReqPtrArgOffset);97 }98 99 return llvm::Error::success();100}101 102// Extract the mapping of host function pointers to device function pointers103// from the entry table. Functions marked as 'indirect' in OpenMP will have104// offloading entries generated for them which map the host's function pointer105// to a global containing the corresponding function pointer on the device.106static llvm::Expected<std::pair<void *, uint64_t>>107setupIndirectCallTable(DeviceTy &Device, __tgt_device_image *Image,108 __tgt_device_binary Binary) {109 AsyncInfoTy AsyncInfo(Device);110 llvm::ArrayRef<llvm::offloading::EntryTy> Entries(Image->EntriesBegin,111 Image->EntriesEnd);112 llvm::SmallVector<std::pair<void *, void *>> IndirectCallTable;113 for (const auto &Entry : Entries) {114 if (Entry.Kind != llvm::object::OffloadKind::OFK_OpenMP ||115 Entry.Size == 0 ||116 (!(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT) &&117 !(Entry.Flags & OMP_DECLARE_TARGET_INDIRECT_VTABLE)))118 continue;119 120 size_t PtrSize = sizeof(void *);121 if (Entry.Flags & OMP_DECLARE_TARGET_INDIRECT_VTABLE) {122 // This is a VTable entry, the current entry is the first index of the123 // VTable and Entry.Size is the total size of the VTable. Unlike the124 // indirect function case below, the Global is not of size Entry.Size and125 // is instead of size PtrSize (sizeof(void*)).126 void *Vtable;127 void *res;128 if (Device.RTL->get_global(Binary, PtrSize, Entry.SymbolName, &Vtable))129 return error::createOffloadError(error::ErrorCode::INVALID_BINARY,130 "failed to load %s", Entry.SymbolName);131 132 // HstPtr = Entry.Address;133 if (Device.retrieveData(&res, Vtable, PtrSize, AsyncInfo))134 return error::createOffloadError(error::ErrorCode::INVALID_BINARY,135 "failed to load %s", Entry.SymbolName);136 if (Device.synchronize(AsyncInfo))137 return error::createOffloadError(138 error::ErrorCode::INVALID_BINARY,139 "failed to synchronize after retrieving %s", Entry.SymbolName);140 // Calculate and emplace entire Vtable from first Vtable byte141 for (uint64_t i = 0; i < Entry.Size / PtrSize; ++i) {142 auto &[HstPtr, DevPtr] = IndirectCallTable.emplace_back();143 HstPtr = reinterpret_cast<void *>(144 reinterpret_cast<uintptr_t>(Entry.Address) + i * PtrSize);145 DevPtr = reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(res) +146 i * PtrSize);147 }148 } else {149 // Indirect function case: Entry.Size should equal PtrSize since we're150 // dealing with a single function pointer (not a VTable)151 assert(Entry.Size == PtrSize && "Global not a function pointer?");152 auto &[HstPtr, DevPtr] = IndirectCallTable.emplace_back();153 void *Ptr;154 if (Device.RTL->get_global(Binary, Entry.Size, Entry.SymbolName, &Ptr))155 return error::createOffloadError(error::ErrorCode::INVALID_BINARY,156 "failed to load %s", Entry.SymbolName);157 158 HstPtr = Entry.Address;159 if (Device.retrieveData(&DevPtr, Ptr, Entry.Size, AsyncInfo))160 return error::createOffloadError(error::ErrorCode::INVALID_BINARY,161 "failed to load %s", Entry.SymbolName);162 }163 if (Device.synchronize(AsyncInfo))164 return error::createOffloadError(165 error::ErrorCode::INVALID_BINARY,166 "failed to synchronize after retrieving %s", Entry.SymbolName);167 }168 169 // If we do not have any indirect globals we exit early.170 if (IndirectCallTable.empty())171 return std::pair{nullptr, 0};172 173 // Sort the array to allow for more efficient lookup of device pointers.174 llvm::sort(IndirectCallTable,175 [](const auto &x, const auto &y) { return x.first < y.first; });176 177 uint64_t TableSize =178 IndirectCallTable.size() * sizeof(std::pair<void *, void *>);179 void *DevicePtr = Device.allocData(TableSize, nullptr, TARGET_ALLOC_DEVICE);180 if (Device.submitData(DevicePtr, IndirectCallTable.data(), TableSize,181 AsyncInfo))182 return error::createOffloadError(error::ErrorCode::INVALID_BINARY,183 "failed to copy data");184 return std::pair<void *, uint64_t>(DevicePtr, IndirectCallTable.size());185}186 187// Load binary to device and perform global initialization if needed.188llvm::Expected<__tgt_device_binary>189DeviceTy::loadBinary(__tgt_device_image *Img) {190 __tgt_device_binary Binary;191 192 if (RTL->load_binary(RTLDeviceID, Img, &Binary) != OFFLOAD_SUCCESS)193 return error::createOffloadError(error::ErrorCode::INVALID_BINARY,194 "failed to load binary %p", Img);195 196 // This symbol is optional.197 void *DeviceEnvironmentPtr;198 if (RTL->get_global(Binary, sizeof(DeviceEnvironmentTy),199 "__omp_rtl_device_environment", &DeviceEnvironmentPtr))200 return Binary;201 202 // Obtain a table mapping host function pointers to device function pointers.203 auto CallTablePairOrErr = setupIndirectCallTable(*this, Img, Binary);204 if (!CallTablePairOrErr)205 return CallTablePairOrErr.takeError();206 207 GenericDeviceTy &GenericDevice = RTL->getDevice(RTLDeviceID);208 DeviceEnvironmentTy DeviceEnvironment;209 DeviceEnvironment.DeviceDebugKind = GenericDevice.getDebugKind();210 DeviceEnvironment.NumDevices = RTL->getNumDevices();211 // TODO: The device ID used here is not the real device ID used by OpenMP.212 DeviceEnvironment.DeviceNum = RTLDeviceID;213 DeviceEnvironment.DynamicMemSize = GenericDevice.getDynamicMemorySize();214 DeviceEnvironment.ClockFrequency = GenericDevice.getClockFrequency();215 DeviceEnvironment.IndirectCallTable =216 reinterpret_cast<uintptr_t>(CallTablePairOrErr->first);217 DeviceEnvironment.IndirectCallTableSize = CallTablePairOrErr->second;218 DeviceEnvironment.HardwareParallelism =219 GenericDevice.getHardwareParallelism();220 221 AsyncInfoTy AsyncInfo(*this);222 if (submitData(DeviceEnvironmentPtr, &DeviceEnvironment,223 sizeof(DeviceEnvironment), AsyncInfo))224 return error::createOffloadError(error::ErrorCode::INVALID_BINARY,225 "failed to copy data");226 227 return Binary;228}229 230void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {231 /// RAII to establish tool anchors before and after data allocation232 void *TargetPtr = nullptr;233 OMPT_IF_BUILT(InterfaceRAII TargetDataAllocRAII(234 RegionInterface.getCallbacks<ompt_target_data_alloc>(),235 DeviceID, HstPtr, &TargetPtr, Size,236 /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)237 238 TargetPtr = RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind);239 return TargetPtr;240}241 242int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) {243 /// RAII to establish tool anchors before and after data deletion244 OMPT_IF_BUILT(InterfaceRAII TargetDataDeleteRAII(245 RegionInterface.getCallbacks<ompt_target_data_delete>(),246 DeviceID, TgtAllocBegin,247 /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)248 249 return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind);250}251 252// Submit data to device253int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,254 AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry,255 MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {256 if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)257 MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/true,258 Entry, HDTTMapPtr);259 260 /// RAII to establish tool anchors before and after data submit261 OMPT_IF_BUILT(262 InterfaceRAII TargetDataSubmitRAII(263 RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>(),264 omp_get_initial_device(), HstPtrBegin, DeviceID, TgtPtrBegin, Size,265 /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)266 267 return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size,268 AsyncInfo);269}270 271// Retrieve data from device272int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,273 int64_t Size, AsyncInfoTy &AsyncInfo,274 HostDataToTargetTy *Entry,275 MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {276 if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)277 MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/false,278 Entry, HDTTMapPtr);279 280 /// RAII to establish tool anchors before and after data retrieval281 OMPT_IF_BUILT(282 InterfaceRAII TargetDataRetrieveRAII(283 RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),284 DeviceID, TgtPtrBegin, omp_get_initial_device(), HstPtrBegin, Size,285 /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)286 287 return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size,288 AsyncInfo);289}290 291// Copy data from current device to destination device directly292int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,293 int64_t Size, AsyncInfoTy &AsyncInfo) {294 /// RAII to establish tool anchors before and after data exchange295 /// Note: Despite the fact that this is a data exchange, we use 'from_device'296 /// operation enum (w.r.t. ompt_target_data_op_t) as there is currently297 /// no better alternative. It is still possible to distinguish this298 /// scenario from a real data retrieve by checking if both involved299 /// device numbers are less than omp_get_num_devices().300 OMPT_IF_BUILT(301 InterfaceRAII TargetDataExchangeRAII(302 RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),303 RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size,304 /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)305 if (!AsyncInfo) {306 return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,307 Size);308 }309 return RTL->data_exchange_async(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID,310 DstPtr, Size, AsyncInfo);311}312 313int32_t DeviceTy::dataFence(AsyncInfoTy &AsyncInfo) {314 return RTL->data_fence(RTLDeviceID, AsyncInfo);315}316 317int32_t DeviceTy::notifyDataMapped(void *HstPtr, int64_t Size) {318 DP("Notifying about new mapping: HstPtr=" DPxMOD ", Size=%" PRId64 "\n",319 DPxPTR(HstPtr), Size);320 321 if (RTL->data_notify_mapped(RTLDeviceID, HstPtr, Size)) {322 REPORT("Notifying about data mapping failed.\n");323 return OFFLOAD_FAIL;324 }325 return OFFLOAD_SUCCESS;326}327 328int32_t DeviceTy::notifyDataUnmapped(void *HstPtr) {329 DP("Notifying about an unmapping: HstPtr=" DPxMOD "\n", DPxPTR(HstPtr));330 331 if (RTL->data_notify_unmapped(RTLDeviceID, HstPtr)) {332 REPORT("Notifying about data unmapping failed.\n");333 return OFFLOAD_FAIL;334 }335 return OFFLOAD_SUCCESS;336}337 338// Run region on device339int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,340 ptrdiff_t *TgtOffsets, KernelArgsTy &KernelArgs,341 AsyncInfoTy &AsyncInfo) {342 return RTL->launch_kernel(RTLDeviceID, TgtEntryPtr, TgtVarsPtr, TgtOffsets,343 &KernelArgs, AsyncInfo);344}345 346// Run region on device347bool DeviceTy::printDeviceInfo() {348 RTL->print_device_info(RTLDeviceID);349 return true;350}351 352// Whether data can be copied to DstDevice directly353bool DeviceTy::isDataExchangable(const DeviceTy &DstDevice) {354 if (RTL != DstDevice.RTL)355 return false;356 357 if (RTL->is_data_exchangable(RTLDeviceID, DstDevice.RTLDeviceID))358 return true;359 return false;360}361 362int32_t DeviceTy::synchronize(AsyncInfoTy &AsyncInfo) {363 return RTL->synchronize(RTLDeviceID, AsyncInfo);364}365 366int32_t DeviceTy::queryAsync(AsyncInfoTy &AsyncInfo) {367 return RTL->query_async(RTLDeviceID, AsyncInfo);368}369 370int32_t DeviceTy::createEvent(void **Event) {371 return RTL->create_event(RTLDeviceID, Event);372}373 374int32_t DeviceTy::recordEvent(void *Event, AsyncInfoTy &AsyncInfo) {375 return RTL->record_event(RTLDeviceID, Event, AsyncInfo);376}377 378int32_t DeviceTy::waitEvent(void *Event, AsyncInfoTy &AsyncInfo) {379 return RTL->wait_event(RTLDeviceID, Event, AsyncInfo);380}381 382int32_t DeviceTy::syncEvent(void *Event) {383 return RTL->sync_event(RTLDeviceID, Event);384}385 386int32_t DeviceTy::destroyEvent(void *Event) {387 return RTL->destroy_event(RTLDeviceID, Event);388}389 390void DeviceTy::dumpOffloadEntries() {391 fprintf(stderr, "Device %i offload entries:\n", DeviceID);392 for (auto &It : *DeviceOffloadEntries.getExclusiveAccessor()) {393 const char *Kind = "kernel";394 if (It.second.isLink())395 Kind = "link";396 else if (It.second.isGlobal())397 Kind = "global var.";398 fprintf(stderr, " %11s: %s\n", Kind, It.second.getNameAsCStr());399 }400}401 402bool DeviceTy::useAutoZeroCopy() {403 if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY)404 return false;405 return RTL->use_auto_zero_copy(RTLDeviceID);406}407 408bool DeviceTy::isAccessiblePtr(const void *Ptr, size_t Size) {409 return RTL->is_accessible_ptr(RTLDeviceID, Ptr, Size);410}411