529 lines · cpp
1//===-RTLs/generic-64bit/src/rtl.cpp - Target RTLs Implementation - C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// RTL NextGen for generic 64-bit machine10//11//===----------------------------------------------------------------------===//12 13#include <cassert>14#include <cstddef>15#include <ffi.h>16#include <string>17#include <unordered_map>18 19#include "Shared/Debug.h"20#include "Shared/Environment.h"21#include "Utils/ELF.h"22 23#include "GlobalHandler.h"24#include "OpenMP/OMPT/Callback.h"25#include "PluginInterface.h"26#include "omptarget.h"27 28#include "llvm/ADT/SmallVector.h"29#include "llvm/Frontend/OpenMP/OMPConstants.h"30#include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"31#include "llvm/Frontend/OpenMP/OMPGridValues.h"32#include "llvm/Support/DynamicLibrary.h"33 34#if !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) || \35 !defined(__ORDER_BIG_ENDIAN__)36#error "Missing preprocessor definitions for endianness detection."37#endif38 39#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)40#define LITTLEENDIAN_CPU41#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)42#define BIGENDIAN_CPU43#endif44 45// The number of devices in this plugin.46#define NUM_DEVICES 447 48namespace llvm {49namespace omp {50namespace target {51namespace plugin {52 53/// Forward declarations for all specialized data structures.54struct GenELF64KernelTy;55struct GenELF64DeviceTy;56struct GenELF64PluginTy;57 58using llvm::sys::DynamicLibrary;59using namespace error;60 61/// Class implementing kernel functionalities for GenELF64.62struct GenELF64KernelTy : public GenericKernelTy {63 /// Construct the kernel with a name and an execution mode.64 GenELF64KernelTy(const char *Name) : GenericKernelTy(Name), Func(nullptr) {}65 66 /// Initialize the kernel.67 Error initImpl(GenericDeviceTy &Device, DeviceImageTy &Image) override {68 // Functions have zero size.69 GlobalTy Global(getName(), 0);70 71 // Get the metadata (address) of the kernel function.72 GenericGlobalHandlerTy &GHandler = Device.Plugin.getGlobalHandler();73 if (auto Err = GHandler.getGlobalMetadataFromDevice(Device, Image, Global))74 return Err;75 76 // Check that the function pointer is valid.77 if (!Global.getPtr())78 return Plugin::error(ErrorCode::INVALID_BINARY,79 "invalid function for kernel %s", getName());80 81 // Save the function pointer.82 Func = (void (*)())Global.getPtr();83 84 KernelEnvironment.Configuration.ExecMode = OMP_TGT_EXEC_MODE_GENERIC;85 KernelEnvironment.Configuration.MayUseNestedParallelism = /*Unknown=*/2;86 KernelEnvironment.Configuration.UseGenericStateMachine = /*Unknown=*/2;87 88 // Set the maximum number of threads to a single.89 MaxNumThreads = 1;90 return Plugin::success();91 }92 93 /// Launch the kernel using the libffi.94 Error launchImpl(GenericDeviceTy &GenericDevice, uint32_t NumThreads[3],95 uint32_t NumBlocks[3], KernelArgsTy &KernelArgs,96 KernelLaunchParamsTy LaunchParams,97 AsyncInfoWrapperTy &AsyncInfoWrapper) const override {98 // Create a vector of ffi_types, one per argument.99 SmallVector<ffi_type *, 16> ArgTypes(KernelArgs.NumArgs, &ffi_type_pointer);100 ffi_type **ArgTypesPtr = (ArgTypes.size()) ? &ArgTypes[0] : nullptr;101 102 // Prepare the cif structure before running the kernel function.103 ffi_cif Cif;104 ffi_status Status = ffi_prep_cif(&Cif, FFI_DEFAULT_ABI, KernelArgs.NumArgs,105 &ffi_type_void, ArgTypesPtr);106 if (Status != FFI_OK)107 return Plugin::error(ErrorCode::UNKNOWN, "error in ffi_prep_cif: %d",108 Status);109 110 // Call the kernel function through libffi.111 long Return;112 ffi_call(&Cif, Func, &Return, (void **)LaunchParams.Ptrs);113 114 return Plugin::success();115 }116 117 /// Return maximum block size for maximum occupancy118 Expected<uint64_t> maxGroupSize(GenericDeviceTy &Device,119 uint64_t DynamicMemSize) const override {120 return Plugin::error(121 ErrorCode::UNSUPPORTED,122 "occupancy calculations are not implemented for the host device");123 }124 125private:126 /// The kernel function to execute.127 void (*Func)(void);128};129 130/// Class implementing the GenELF64 device images properties.131struct GenELF64DeviceImageTy : public DeviceImageTy {132 /// Create the GenELF64 image with the id and the target image pointer.133 GenELF64DeviceImageTy(int32_t ImageId, GenericDeviceTy &Device,134 std::unique_ptr<MemoryBuffer> &&TgtImage)135 : DeviceImageTy(ImageId, Device, std::move(TgtImage)), DynLib() {}136 137 /// Getter and setter for the dynamic library.138 DynamicLibrary &getDynamicLibrary() { return DynLib; }139 void setDynamicLibrary(const DynamicLibrary &Lib) { DynLib = Lib; }140 141private:142 /// The dynamic library that loaded the image.143 DynamicLibrary DynLib;144};145 146/// Class implementing the device functionalities for GenELF64.147struct GenELF64DeviceTy : public GenericDeviceTy {148 /// Create the device with a specific id.149 GenELF64DeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,150 int32_t NumDevices)151 : GenericDeviceTy(Plugin, DeviceId, NumDevices, GenELF64GridValues) {}152 153 ~GenELF64DeviceTy() {}154 155 /// Initialize the device, which is a no-op156 Error initImpl(GenericPluginTy &Plugin) override { return Plugin::success(); }157 158 /// Unload the binary image159 ///160 /// TODO: This currently does nothing, and should be implemented as part of161 /// broader memory handling logic for this plugin162 Error unloadBinaryImpl(DeviceImageTy *Image) override {163 auto Elf = reinterpret_cast<GenELF64DeviceImageTy *>(Image);164 DynamicLibrary::closeLibrary(Elf->getDynamicLibrary());165 Plugin.free(Elf);166 return Plugin::success();167 }168 169 /// Deinitialize the device, which is a no-op170 Error deinitImpl() override { return Plugin::success(); }171 172 /// See GenericDeviceTy::getComputeUnitKind().173 std::string getComputeUnitKind() const override { return "generic-64bit"; }174 175 /// Construct the kernel for a specific image on the device.176 Expected<GenericKernelTy &> constructKernel(const char *Name) override {177 // Allocate and construct the kernel.178 GenELF64KernelTy *GenELF64Kernel = Plugin.allocate<GenELF64KernelTy>();179 if (!GenELF64Kernel)180 return Plugin::error(ErrorCode::OUT_OF_RESOURCES,181 "failed to allocate memory for GenELF64 kernel");182 183 new (GenELF64Kernel) GenELF64KernelTy(Name);184 185 return *GenELF64Kernel;186 }187 188 /// Set the current context to this device, which is a no-op.189 Error setContext() override { return Plugin::success(); }190 191 /// Load the binary image into the device and allocate an image object.192 Expected<DeviceImageTy *>193 loadBinaryImpl(std::unique_ptr<MemoryBuffer> &&TgtImage,194 int32_t ImageId) override {195 // Allocate and initialize the image object.196 GenELF64DeviceImageTy *Image = Plugin.allocate<GenELF64DeviceImageTy>();197 new (Image) GenELF64DeviceImageTy(ImageId, *this, std::move(TgtImage));198 199 // Create a temporary file.200 char TmpFileName[] = "/tmp/tmpfile_XXXXXX";201 int TmpFileFd = mkstemp(TmpFileName);202 if (TmpFileFd == -1)203 return Plugin::error(ErrorCode::HOST_IO,204 "failed to create tmpfile for loading target image");205 206 // Open the temporary file.207 FILE *TmpFile = fdopen(TmpFileFd, "wb");208 if (!TmpFile)209 return Plugin::error(ErrorCode::HOST_IO,210 "failed to open tmpfile %s for loading target image",211 TmpFileName);212 213 // Write the image into the temporary file.214 size_t Written = fwrite(Image->getStart(), Image->getSize(), 1, TmpFile);215 if (Written != 1)216 return Plugin::error(ErrorCode::HOST_IO,217 "failed to write target image to tmpfile %s",218 TmpFileName);219 220 // Close the temporary file.221 int Ret = fclose(TmpFile);222 if (Ret)223 return Plugin::error(ErrorCode::HOST_IO,224 "failed to close tmpfile %s with the target image",225 TmpFileName);226 227 // Load the temporary file as a dynamic library.228 std::string ErrMsg;229 DynamicLibrary DynLib = DynamicLibrary::getLibrary(TmpFileName, &ErrMsg);230 231 // Check if the loaded library is valid.232 if (!DynLib.isValid())233 return Plugin::error(ErrorCode::INVALID_BINARY,234 "failed to load target image: %s", ErrMsg.c_str());235 236 // Save a reference of the image's dynamic library.237 Image->setDynamicLibrary(DynLib);238 239 return Image;240 }241 242 /// Allocate memory. Use std::malloc in all cases.243 Expected<void *> allocate(size_t Size, void *, TargetAllocTy Kind) override {244 if (Size == 0)245 return nullptr;246 247 void *MemAlloc = nullptr;248 switch (Kind) {249 case TARGET_ALLOC_DEFAULT:250 case TARGET_ALLOC_DEVICE:251 case TARGET_ALLOC_HOST:252 case TARGET_ALLOC_SHARED:253 MemAlloc = std::malloc(Size);254 break;255 }256 return MemAlloc;257 }258 259 /// Free the memory. Use std::free in all cases.260 Error free(void *TgtPtr, TargetAllocTy Kind) override {261 std::free(TgtPtr);262 return Plugin::success();263 }264 265 /// This plugin does nothing to lock buffers. Do not return an error, just266 /// return the same pointer as the device pointer.267 Expected<void *> dataLockImpl(void *HstPtr, int64_t Size) override {268 return HstPtr;269 }270 271 /// Nothing to do when unlocking the buffer.272 Error dataUnlockImpl(void *HstPtr) override { return Plugin::success(); }273 274 /// Indicate that the buffer is not pinned.275 Expected<bool> isPinnedPtrImpl(void *HstPtr, void *&BaseHstPtr,276 void *&BaseDevAccessiblePtr,277 size_t &BaseSize) const override {278 return false;279 }280 281 /// Submit data to the device (host to device transfer).282 Error dataSubmitImpl(void *TgtPtr, const void *HstPtr, int64_t Size,283 AsyncInfoWrapperTy &AsyncInfoWrapper) override {284 std::memcpy(TgtPtr, HstPtr, Size);285 return Plugin::success();286 }287 288 /// Retrieve data from the device (device to host transfer).289 Error dataRetrieveImpl(void *HstPtr, const void *TgtPtr, int64_t Size,290 AsyncInfoWrapperTy &AsyncInfoWrapper) override {291 std::memcpy(HstPtr, TgtPtr, Size);292 return Plugin::success();293 }294 295 /// Exchange data between two devices within the plugin. This function is not296 /// supported in this plugin.297 Error dataExchangeImpl(const void *SrcPtr, GenericDeviceTy &DstGenericDevice,298 void *DstPtr, int64_t Size,299 AsyncInfoWrapperTy &AsyncInfoWrapper) override {300 // This function should never be called because the function301 // GenELF64PluginTy::isDataExchangable() returns false.302 return Plugin::error(ErrorCode::UNSUPPORTED,303 "dataExchangeImpl not supported");304 }305 306 /// Insert a data fence between previous data operations and the following307 /// operations. This is a no-op for Host devices as operations inserted into308 /// a queue are in-order.309 Error dataFence(__tgt_async_info *Async) override {310 return Plugin::success();311 }312 313 Error dataFillImpl(void *TgtPtr, const void *PatternPtr, int64_t PatternSize,314 int64_t Size,315 AsyncInfoWrapperTy &AsyncInfoWrapper) override {316 if (PatternSize == 1) {317 std::memset(TgtPtr, *static_cast<const char *>(PatternPtr), Size);318 } else {319 for (unsigned int Step = 0; Step < Size; Step += PatternSize) {320 auto *Dst = static_cast<char *>(TgtPtr) + Step;321 std::memcpy(Dst, PatternPtr, PatternSize);322 }323 }324 325 return Plugin::success();326 }327 328 /// All functions are already synchronous. No need to do anything on this329 /// synchronization function.330 Error synchronizeImpl(__tgt_async_info &AsyncInfo,331 bool ReleaseQueue) override {332 return Plugin::success();333 }334 335 /// All functions are already synchronous. No need to do anything on this336 /// query function.337 Error queryAsyncImpl(__tgt_async_info &AsyncInfo) override {338 return Plugin::success();339 }340 341 /// This plugin does not support interoperability342 Error initAsyncInfoImpl(AsyncInfoWrapperTy &AsyncInfoWrapper) override {343 return Plugin::error(ErrorCode::UNSUPPORTED,344 "initAsyncInfoImpl not supported");345 }346 347 Error enqueueHostCallImpl(void (*Callback)(void *), void *UserData,348 AsyncInfoWrapperTy &AsyncInfo) override {349 Callback(UserData);350 return Plugin::success();351 };352 353 /// This plugin does not support the event API. Do nothing without failing.354 Error createEventImpl(void **EventPtrStorage) override {355 *EventPtrStorage = nullptr;356 return Plugin::success();357 }358 Error destroyEventImpl(void *EventPtr) override { return Plugin::success(); }359 Error recordEventImpl(void *EventPtr,360 AsyncInfoWrapperTy &AsyncInfoWrapper) override {361 return Plugin::success();362 }363 Error waitEventImpl(void *EventPtr,364 AsyncInfoWrapperTy &AsyncInfoWrapper) override {365 return Plugin::success();366 }367 Expected<bool> hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfo) override {368 return true;369 }370 Expected<bool> isEventCompleteImpl(void *Event,371 AsyncInfoWrapperTy &AsyncInfo) override {372 return true;373 }374 Error syncEventImpl(void *EventPtr) override { return Plugin::success(); }375 376 /// Print information about the device.377 Expected<InfoTreeNode> obtainInfoImpl() override {378 InfoTreeNode Info;379 Info.add("Device Type", "Generic-elf-64bit");380 return Info;381 }382 383 /// Getters and setters for stack size and heap size not relevant.384 Error getDeviceStackSize(uint64_t &Value) override {385 Value = 0;386 return Plugin::success();387 }388 Error setDeviceStackSize(uint64_t Value) override {389 return Plugin::success();390 }391 392private:393 /// Grid values for Generic ELF64 plugins.394 static constexpr GV GenELF64GridValues = {395 1, // GV_Slot_Size396 1, // GV_Warp_Size397 1, // GV_Max_Teams398 1, // GV_Default_Num_Teams399 1, // GV_SimpleBufferSize400 1, // GV_Max_WG_Size401 1, // GV_Default_WG_Size402 };403};404 405class GenELF64GlobalHandlerTy final : public GenericGlobalHandlerTy {406public:407 Error getGlobalMetadataFromDevice(GenericDeviceTy &GenericDevice,408 DeviceImageTy &Image,409 GlobalTy &DeviceGlobal) override {410 const char *GlobalName = DeviceGlobal.getName().data();411 GenELF64DeviceImageTy &GenELF64Image =412 static_cast<GenELF64DeviceImageTy &>(Image);413 414 // Get dynamic library that has loaded the device image.415 DynamicLibrary &DynLib = GenELF64Image.getDynamicLibrary();416 417 // Get the address of the symbol.418 void *Addr = DynLib.getAddressOfSymbol(GlobalName);419 if (Addr == nullptr) {420 return Plugin::error(ErrorCode::NOT_FOUND, "failed to load global '%s'",421 GlobalName);422 }423 424 // Save the pointer to the symbol.425 DeviceGlobal.setPtr(Addr);426 427 return Plugin::success();428 }429};430 431/// Class implementing the plugin functionalities for GenELF64.432struct GenELF64PluginTy final : public GenericPluginTy {433 /// Create the GenELF64 plugin.434 GenELF64PluginTy() : GenericPluginTy(getTripleArch()) {}435 436 /// This class should not be copied.437 GenELF64PluginTy(const GenELF64PluginTy &) = delete;438 GenELF64PluginTy(GenELF64PluginTy &&) = delete;439 440 /// Initialize the plugin and return the number of devices.441 Expected<int32_t> initImpl() override {442#ifdef USES_DYNAMIC_FFI443 if (auto Err = Plugin::check(ffi_init(), "failed to initialize libffi"))444 return std::move(Err);445#endif446 ODBG("Init") << "GenELF64 plugin detected " << ODBG_IF_LEVEL(2)447 << NUM_DEVICES << " " << ODBG_RESET_LEVEL() << "devices";448 449 return NUM_DEVICES;450 }451 452 /// Deinitialize the plugin.453 Error deinitImpl() override { return Plugin::success(); }454 455 /// Creates a generic ELF device.456 GenericDeviceTy *createDevice(GenericPluginTy &Plugin, int32_t DeviceId,457 int32_t NumDevices) override {458 return new GenELF64DeviceTy(Plugin, DeviceId, NumDevices);459 }460 461 /// Creates a generic global handler.462 GenericGlobalHandlerTy *createGlobalHandler() override {463 return new GenELF64GlobalHandlerTy();464 }465 466 /// Get the ELF code to recognize the compatible binary images.467 uint16_t getMagicElfBits() const override {468 return utils::elf::getTargetMachine();469 }470 471 /// This plugin does not support exchanging data between two devices.472 bool isDataExchangable(int32_t SrcDeviceId, int32_t DstDeviceId) override {473 return false;474 }475 476 /// All images (ELF-compatible) should be compatible with this plugin.477 Expected<bool> isELFCompatible(uint32_t, StringRef) const override {478 return true;479 }480 481 Triple::ArchType getTripleArch() const override {482#if defined(__x86_64__)483 return llvm::Triple::x86_64;484#elif defined(__s390x__)485 return llvm::Triple::systemz;486#elif defined(__aarch64__)487#ifdef LITTLEENDIAN_CPU488 return llvm::Triple::aarch64;489#else490 return llvm::Triple::aarch64_be;491#endif492#elif defined(__powerpc64__)493#ifdef LITTLEENDIAN_CPU494 return llvm::Triple::ppc64le;495#else496 return llvm::Triple::ppc64;497#endif498#elif defined(__riscv) && (__riscv_xlen == 64)499 return llvm::Triple::riscv64;500#elif defined(__loongarch__) && (__loongarch_grlen == 64)501 return llvm::Triple::loongarch64;502#else503 return llvm::Triple::UnknownArch;504#endif505 }506 507 const char *getName() const override { return GETNAME(TARGET_NAME); }508};509 510template <typename... ArgsTy>511static Error Plugin::check(int32_t Code, const char *ErrMsg, ArgsTy... Args) {512 if (Code == 0)513 return Plugin::success();514 515 return Plugin::error(ErrorCode::UNKNOWN, ErrMsg, Args...,516 std::to_string(Code).data());517}518 519} // namespace plugin520} // namespace target521} // namespace omp522} // namespace llvm523 524extern "C" {525llvm::omp::target::plugin::GenericPluginTy *createPlugin_host() {526 return new llvm::omp::target::plugin::GenELF64PluginTy();527}528}529