1087 lines · cpp
1//===- OffloadWrapper.cpp ---------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/Frontend/Offloading/OffloadWrapper.h"10#include "llvm/ADT/ArrayRef.h"11#include "llvm/ADT/SmallVector.h"12#include "llvm/ADT/StringRef.h"13#include "llvm/ADT/Twine.h"14#include "llvm/BinaryFormat/Magic.h"15#include "llvm/Frontend/Offloading/Utility.h"16#include "llvm/IR/Constants.h"17#include "llvm/IR/DerivedTypes.h"18#include "llvm/IR/GlobalVariable.h"19#include "llvm/IR/IRBuilder.h"20#include "llvm/IR/LLVMContext.h"21#include "llvm/IR/Module.h"22#include "llvm/IR/Type.h"23#include "llvm/Object/OffloadBinary.h"24#include "llvm/Support/Error.h"25#include "llvm/Support/ErrorHandling.h"26#include "llvm/Support/LineIterator.h"27#include "llvm/Support/MemoryBufferRef.h"28#include "llvm/TargetParser/Triple.h"29#include "llvm/Transforms/Utils/ModuleUtils.h"30 31#include <memory>32#include <utility>33 34using namespace llvm;35using namespace llvm::object;36using namespace llvm::offloading;37 38namespace {39/// Magic number that begins the section containing the CUDA fatbinary.40constexpr unsigned CudaFatMagic = 0x466243b1;41constexpr unsigned HIPFatMagic = 0x48495046;42 43IntegerType *getSizeTTy(Module &M) {44 return M.getDataLayout().getIntPtrType(M.getContext());45}46 47// struct __tgt_device_image {48// void *ImageStart;49// void *ImageEnd;50// __tgt_offload_entry *EntriesBegin;51// __tgt_offload_entry *EntriesEnd;52// };53StructType *getDeviceImageTy(Module &M) {54 LLVMContext &C = M.getContext();55 StructType *ImageTy = StructType::getTypeByName(C, "__tgt_device_image");56 if (!ImageTy)57 ImageTy =58 StructType::create("__tgt_device_image", PointerType::getUnqual(C),59 PointerType::getUnqual(C), PointerType::getUnqual(C),60 PointerType::getUnqual(C));61 return ImageTy;62}63 64PointerType *getDeviceImagePtrTy(Module &M) {65 return PointerType::getUnqual(M.getContext());66}67 68// struct __tgt_bin_desc {69// int32_t NumDeviceImages;70// __tgt_device_image *DeviceImages;71// __tgt_offload_entry *HostEntriesBegin;72// __tgt_offload_entry *HostEntriesEnd;73// };74StructType *getBinDescTy(Module &M) {75 LLVMContext &C = M.getContext();76 StructType *DescTy = StructType::getTypeByName(C, "__tgt_bin_desc");77 if (!DescTy)78 DescTy = StructType::create(79 "__tgt_bin_desc", Type::getInt32Ty(C), getDeviceImagePtrTy(M),80 PointerType::getUnqual(C), PointerType::getUnqual(C));81 return DescTy;82}83 84PointerType *getBinDescPtrTy(Module &M) {85 return PointerType::getUnqual(M.getContext());86}87 88/// Creates binary descriptor for the given device images. Binary descriptor89/// is an object that is passed to the offloading runtime at program startup90/// and it describes all device images available in the executable or shared91/// library. It is defined as follows92///93/// __attribute__((visibility("hidden")))94/// extern __tgt_offload_entry *__start_omp_offloading_entries;95/// __attribute__((visibility("hidden")))96/// extern __tgt_offload_entry *__stop_omp_offloading_entries;97///98/// static const char Image0[] = { <Bufs.front() contents> };99/// ...100/// static const char ImageN[] = { <Bufs.back() contents> };101///102/// static const __tgt_device_image Images[] = {103/// {104/// Image0, /*ImageStart*/105/// Image0 + sizeof(Image0), /*ImageEnd*/106/// __start_omp_offloading_entries, /*EntriesBegin*/107/// __stop_omp_offloading_entries /*EntriesEnd*/108/// },109/// ...110/// {111/// ImageN, /*ImageStart*/112/// ImageN + sizeof(ImageN), /*ImageEnd*/113/// __start_omp_offloading_entries, /*EntriesBegin*/114/// __stop_omp_offloading_entries /*EntriesEnd*/115/// }116/// };117///118/// static const __tgt_bin_desc BinDesc = {119/// sizeof(Images) / sizeof(Images[0]), /*NumDeviceImages*/120/// Images, /*DeviceImages*/121/// __start_omp_offloading_entries, /*HostEntriesBegin*/122/// __stop_omp_offloading_entries /*HostEntriesEnd*/123/// };124///125/// Global variable that represents BinDesc is returned.126GlobalVariable *createBinDesc(Module &M, ArrayRef<ArrayRef<char>> Bufs,127 EntryArrayTy EntryArray, StringRef Suffix,128 bool Relocatable) {129 LLVMContext &C = M.getContext();130 auto [EntriesB, EntriesE] = EntryArray;131 132 auto *Zero = ConstantInt::get(getSizeTTy(M), 0u);133 Constant *ZeroZero[] = {Zero, Zero};134 135 // Create initializer for the images array.136 SmallVector<Constant *, 4u> ImagesInits;137 ImagesInits.reserve(Bufs.size());138 for (ArrayRef<char> Buf : Bufs) {139 // We embed the full offloading entry so the binary utilities can parse it.140 auto *Data = ConstantDataArray::get(C, Buf);141 auto *Image = new GlobalVariable(M, Data->getType(), /*isConstant=*/true,142 GlobalVariable::InternalLinkage, Data,143 ".omp_offloading.device_image" + Suffix);144 Image->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);145 Image->setSection(Relocatable ? ".llvm.offloading.relocatable"146 : ".llvm.offloading");147 Image->setAlignment(Align(object::OffloadBinary::getAlignment()));148 149 StringRef Binary(Buf.data(), Buf.size());150 151 uint64_t BeginOffset = 0;152 uint64_t EndOffset = Binary.size();153 154 // Optionally use an offload binary for its offload dumping support.155 // The device image struct contains the pointer to the beginning and end of156 // the image stored inside of the offload binary. There should only be one157 // of these for each buffer so we parse it out manually.158 if (identify_magic(Binary) == file_magic::offload_binary) {159 const auto *Header =160 reinterpret_cast<const object::OffloadBinary::Header *>(161 Binary.bytes_begin());162 const auto *Entry =163 reinterpret_cast<const object::OffloadBinary::Entry *>(164 Binary.bytes_begin() + Header->EntryOffset);165 BeginOffset = Entry->ImageOffset;166 EndOffset = Entry->ImageOffset + Entry->ImageSize;167 }168 169 auto *Begin = ConstantInt::get(getSizeTTy(M), BeginOffset);170 auto *Size = ConstantInt::get(getSizeTTy(M), EndOffset);171 Constant *ZeroBegin[] = {Zero, Begin};172 Constant *ZeroSize[] = {Zero, Size};173 174 auto *ImageB =175 ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroBegin);176 auto *ImageE =177 ConstantExpr::getGetElementPtr(Image->getValueType(), Image, ZeroSize);178 179 ImagesInits.push_back(ConstantStruct::get(getDeviceImageTy(M), ImageB,180 ImageE, EntriesB, EntriesE));181 }182 183 // Then create images array.184 auto *ImagesData = ConstantArray::get(185 ArrayType::get(getDeviceImageTy(M), ImagesInits.size()), ImagesInits);186 187 auto *Images =188 new GlobalVariable(M, ImagesData->getType(), /*isConstant*/ true,189 GlobalValue::InternalLinkage, ImagesData,190 ".omp_offloading.device_images" + Suffix);191 Images->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);192 193 auto *ImagesB =194 ConstantExpr::getGetElementPtr(Images->getValueType(), Images, ZeroZero);195 196 // And finally create the binary descriptor object.197 auto *DescInit = ConstantStruct::get(198 getBinDescTy(M),199 ConstantInt::get(Type::getInt32Ty(C), ImagesInits.size()), ImagesB,200 EntriesB, EntriesE);201 202 return new GlobalVariable(M, DescInit->getType(), /*isConstant=*/true,203 GlobalValue::InternalLinkage, DescInit,204 ".omp_offloading.descriptor" + Suffix);205}206 207Function *createUnregisterFunction(Module &M, GlobalVariable *BinDesc,208 StringRef Suffix) {209 LLVMContext &C = M.getContext();210 auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);211 auto *Func =212 Function::Create(FuncTy, GlobalValue::InternalLinkage,213 ".omp_offloading.descriptor_unreg" + Suffix, &M);214 Func->setSection(".text.startup");215 216 // Get __tgt_unregister_lib function declaration.217 auto *UnRegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M),218 /*isVarArg*/ false);219 FunctionCallee UnRegFuncC =220 M.getOrInsertFunction("__tgt_unregister_lib", UnRegFuncTy);221 222 // Construct function body223 IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));224 Builder.CreateCall(UnRegFuncC, BinDesc);225 Builder.CreateRetVoid();226 227 return Func;228}229 230void createRegisterFunction(Module &M, GlobalVariable *BinDesc,231 StringRef Suffix) {232 LLVMContext &C = M.getContext();233 auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);234 auto *Func = Function::Create(FuncTy, GlobalValue::InternalLinkage,235 ".omp_offloading.descriptor_reg" + Suffix, &M);236 Func->setSection(".text.startup");237 238 // Get __tgt_register_lib function declaration.239 auto *RegFuncTy = FunctionType::get(Type::getVoidTy(C), getBinDescPtrTy(M),240 /*isVarArg*/ false);241 FunctionCallee RegFuncC =242 M.getOrInsertFunction("__tgt_register_lib", RegFuncTy);243 244 auto *AtExitTy = FunctionType::get(245 Type::getInt32Ty(C), PointerType::getUnqual(C), /*isVarArg=*/false);246 FunctionCallee AtExit = M.getOrInsertFunction("atexit", AtExitTy);247 248 Function *UnregFunc = createUnregisterFunction(M, BinDesc, Suffix);249 250 // Construct function body251 IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));252 253 Builder.CreateCall(RegFuncC, BinDesc);254 255 // Register the destructors with 'atexit'. This is expected by the CUDA256 // runtime and ensures that we clean up before dynamic objects are destroyed.257 // This needs to be done after plugin initialization to ensure that it is258 // called before the plugin runtime is destroyed.259 Builder.CreateCall(AtExit, UnregFunc);260 Builder.CreateRetVoid();261 262 // Add this function to constructors.263 appendToGlobalCtors(M, Func, /*Priority=*/101);264}265 266// struct fatbin_wrapper {267// int32_t magic;268// int32_t version;269// void *image;270// void *reserved;271//};272StructType *getFatbinWrapperTy(Module &M) {273 LLVMContext &C = M.getContext();274 StructType *FatbinTy = StructType::getTypeByName(C, "fatbin_wrapper");275 if (!FatbinTy)276 FatbinTy = StructType::create(277 "fatbin_wrapper", Type::getInt32Ty(C), Type::getInt32Ty(C),278 PointerType::getUnqual(C), PointerType::getUnqual(C));279 return FatbinTy;280}281 282/// Embed the image \p Image into the module \p M so it can be found by the283/// runtime.284GlobalVariable *createFatbinDesc(Module &M, ArrayRef<char> Image, bool IsHIP,285 StringRef Suffix) {286 LLVMContext &C = M.getContext();287 llvm::Type *Int8PtrTy = PointerType::getUnqual(C);288 const llvm::Triple &Triple = M.getTargetTriple();289 290 // Create the global string containing the fatbinary.291 StringRef FatbinConstantSection =292 IsHIP ? ".hip_fatbin"293 : (Triple.isMacOSX() ? "__NV_CUDA,__nv_fatbin" : ".nv_fatbin");294 auto *Data = ConstantDataArray::get(C, Image);295 auto *Fatbin = new GlobalVariable(M, Data->getType(), /*isConstant*/ true,296 GlobalVariable::InternalLinkage, Data,297 ".fatbin_image" + Suffix);298 Fatbin->setSection(FatbinConstantSection);299 300 // Create the fatbinary wrapper301 StringRef FatbinWrapperSection = IsHIP ? ".hipFatBinSegment"302 : Triple.isMacOSX() ? "__NV_CUDA,__fatbin"303 : ".nvFatBinSegment";304 Constant *FatbinWrapper[] = {305 ConstantInt::get(Type::getInt32Ty(C), IsHIP ? HIPFatMagic : CudaFatMagic),306 ConstantInt::get(Type::getInt32Ty(C), 1),307 ConstantExpr::getPointerBitCastOrAddrSpaceCast(Fatbin, Int8PtrTy),308 ConstantPointerNull::get(PointerType::getUnqual(C))};309 310 Constant *FatbinInitializer =311 ConstantStruct::get(getFatbinWrapperTy(M), FatbinWrapper);312 313 auto *FatbinDesc =314 new GlobalVariable(M, getFatbinWrapperTy(M),315 /*isConstant*/ true, GlobalValue::InternalLinkage,316 FatbinInitializer, ".fatbin_wrapper" + Suffix);317 FatbinDesc->setSection(FatbinWrapperSection);318 FatbinDesc->setAlignment(Align(8));319 320 return FatbinDesc;321}322 323/// Create the register globals function. We will iterate all of the offloading324/// entries stored at the begin / end symbols and register them according to325/// their type. This creates the following function in IR:326///327/// extern struct __tgt_offload_entry __start_cuda_offloading_entries;328/// extern struct __tgt_offload_entry __stop_cuda_offloading_entries;329///330/// extern void __cudaRegisterFunction(void **, void *, void *, void *, int,331/// void *, void *, void *, void *, int *);332/// extern void __cudaRegisterVar(void **, void *, void *, void *, int32_t,333/// int64_t, int32_t, int32_t);334///335/// void __cudaRegisterTest(void **fatbinHandle) {336/// for (struct __tgt_offload_entry *entry = &__start_cuda_offloading_entries;337/// entry != &__stop_cuda_offloading_entries; ++entry) {338/// if (entry->Kind != OFK_CUDA)339/// continue340///341/// if (!entry->Size)342/// __cudaRegisterFunction(fatbinHandle, entry->addr, entry->name,343/// entry->name, -1, 0, 0, 0, 0, 0);344/// else345/// __cudaRegisterVar(fatbinHandle, entry->addr, entry->name, entry->name,346/// 0, entry->size, 0, 0);347/// }348/// }349Function *createRegisterGlobalsFunction(Module &M, bool IsHIP,350 EntryArrayTy EntryArray,351 StringRef Suffix,352 bool EmitSurfacesAndTextures) {353 LLVMContext &C = M.getContext();354 auto [EntriesB, EntriesE] = EntryArray;355 356 // Get the __cudaRegisterFunction function declaration.357 PointerType *Int8PtrTy = PointerType::get(C, 0);358 PointerType *Int8PtrPtrTy = PointerType::get(C, 0);359 PointerType *Int32PtrTy = PointerType::get(C, 0);360 auto *RegFuncTy = FunctionType::get(361 Type::getInt32Ty(C),362 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),363 Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int32PtrTy},364 /*isVarArg*/ false);365 FunctionCallee RegFunc = M.getOrInsertFunction(366 IsHIP ? "__hipRegisterFunction" : "__cudaRegisterFunction", RegFuncTy);367 368 // Get the __cudaRegisterVar function declaration.369 auto *RegVarTy = FunctionType::get(370 Type::getVoidTy(C),371 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),372 getSizeTTy(M), Type::getInt32Ty(C), Type::getInt32Ty(C)},373 /*isVarArg*/ false);374 FunctionCallee RegVar = M.getOrInsertFunction(375 IsHIP ? "__hipRegisterVar" : "__cudaRegisterVar", RegVarTy);376 377 // Get the __cudaRegisterSurface function declaration.378 FunctionType *RegManagedVarTy =379 FunctionType::get(Type::getVoidTy(C),380 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,381 getSizeTTy(M), Type::getInt32Ty(C)},382 /*isVarArg=*/false);383 FunctionCallee RegManagedVar = M.getOrInsertFunction(384 IsHIP ? "__hipRegisterManagedVar" : "__cudaRegisterManagedVar",385 RegManagedVarTy);386 387 // Get the __cudaRegisterSurface function declaration.388 FunctionType *RegSurfaceTy =389 FunctionType::get(Type::getVoidTy(C),390 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy,391 Type::getInt32Ty(C), Type::getInt32Ty(C)},392 /*isVarArg=*/false);393 FunctionCallee RegSurface = M.getOrInsertFunction(394 IsHIP ? "__hipRegisterSurface" : "__cudaRegisterSurface", RegSurfaceTy);395 396 // Get the __cudaRegisterTexture function declaration.397 FunctionType *RegTextureTy = FunctionType::get(398 Type::getVoidTy(C),399 {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C),400 Type::getInt32Ty(C), Type::getInt32Ty(C)},401 /*isVarArg=*/false);402 FunctionCallee RegTexture = M.getOrInsertFunction(403 IsHIP ? "__hipRegisterTexture" : "__cudaRegisterTexture", RegTextureTy);404 405 auto *RegGlobalsTy = FunctionType::get(Type::getVoidTy(C), Int8PtrPtrTy,406 /*isVarArg*/ false);407 auto *RegGlobalsFn =408 Function::Create(RegGlobalsTy, GlobalValue::InternalLinkage,409 IsHIP ? ".hip.globals_reg" : ".cuda.globals_reg", &M);410 RegGlobalsFn->setSection(".text.startup");411 412 // Create the loop to register all the entries.413 IRBuilder<> Builder(BasicBlock::Create(C, "entry", RegGlobalsFn));414 auto *EntryBB = BasicBlock::Create(C, "while.entry", RegGlobalsFn);415 auto *IfKindBB = BasicBlock::Create(C, "if.kind", RegGlobalsFn);416 auto *IfThenBB = BasicBlock::Create(C, "if.then", RegGlobalsFn);417 auto *IfElseBB = BasicBlock::Create(C, "if.else", RegGlobalsFn);418 auto *SwGlobalBB = BasicBlock::Create(C, "sw.global", RegGlobalsFn);419 auto *SwManagedBB = BasicBlock::Create(C, "sw.managed", RegGlobalsFn);420 auto *SwSurfaceBB = BasicBlock::Create(C, "sw.surface", RegGlobalsFn);421 auto *SwTextureBB = BasicBlock::Create(C, "sw.texture", RegGlobalsFn);422 auto *IfEndBB = BasicBlock::Create(C, "if.end", RegGlobalsFn);423 auto *ExitBB = BasicBlock::Create(C, "while.end", RegGlobalsFn);424 425 auto *EntryCmp = Builder.CreateICmpNE(EntriesB, EntriesE);426 Builder.CreateCondBr(EntryCmp, EntryBB, ExitBB);427 Builder.SetInsertPoint(EntryBB);428 auto *Entry = Builder.CreatePHI(PointerType::getUnqual(C), 2, "entry");429 auto *AddrPtr =430 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,431 {ConstantInt::get(Type::getInt32Ty(C), 0),432 ConstantInt::get(Type::getInt32Ty(C), 4)});433 auto *Addr = Builder.CreateLoad(Int8PtrTy, AddrPtr, "addr");434 auto *AuxAddrPtr =435 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,436 {ConstantInt::get(Type::getInt32Ty(C), 0),437 ConstantInt::get(Type::getInt32Ty(C), 8)});438 auto *AuxAddr = Builder.CreateLoad(Int8PtrTy, AuxAddrPtr, "aux_addr");439 auto *KindPtr =440 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,441 {ConstantInt::get(Type::getInt32Ty(C), 0),442 ConstantInt::get(Type::getInt32Ty(C), 2)});443 auto *Kind = Builder.CreateLoad(Type::getInt16Ty(C), KindPtr, "kind");444 auto *NamePtr =445 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,446 {ConstantInt::get(Type::getInt32Ty(C), 0),447 ConstantInt::get(Type::getInt32Ty(C), 5)});448 auto *Name = Builder.CreateLoad(Int8PtrTy, NamePtr, "name");449 auto *SizePtr =450 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,451 {ConstantInt::get(Type::getInt32Ty(C), 0),452 ConstantInt::get(Type::getInt32Ty(C), 6)});453 auto *Size = Builder.CreateLoad(Type::getInt64Ty(C), SizePtr, "size");454 auto *FlagsPtr =455 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,456 {ConstantInt::get(Type::getInt32Ty(C), 0),457 ConstantInt::get(Type::getInt32Ty(C), 3)});458 auto *Flags = Builder.CreateLoad(Type::getInt32Ty(C), FlagsPtr, "flags");459 auto *DataPtr =460 Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry,461 {ConstantInt::get(Type::getInt32Ty(C), 0),462 ConstantInt::get(Type::getInt32Ty(C), 7)});463 auto *Data = Builder.CreateTrunc(464 Builder.CreateLoad(Type::getInt64Ty(C), DataPtr, "data"),465 Type::getInt32Ty(C));466 auto *Type = Builder.CreateAnd(467 Flags, ConstantInt::get(Type::getInt32Ty(C), 0x7), "type");468 469 // Extract the flags stored in the bit-field and convert them to C booleans.470 auto *ExternBit = Builder.CreateAnd(471 Flags, ConstantInt::get(Type::getInt32Ty(C),472 llvm::offloading::OffloadGlobalExtern));473 auto *Extern = Builder.CreateLShr(474 ExternBit, ConstantInt::get(Type::getInt32Ty(C), 3), "extern");475 auto *ConstantBit = Builder.CreateAnd(476 Flags, ConstantInt::get(Type::getInt32Ty(C),477 llvm::offloading::OffloadGlobalConstant));478 auto *Const = Builder.CreateLShr(479 ConstantBit, ConstantInt::get(Type::getInt32Ty(C), 4), "constant");480 auto *NormalizedBit = Builder.CreateAnd(481 Flags, ConstantInt::get(Type::getInt32Ty(C),482 llvm::offloading::OffloadGlobalNormalized));483 auto *Normalized = Builder.CreateLShr(484 NormalizedBit, ConstantInt::get(Type::getInt32Ty(C), 5), "normalized");485 auto *KindCond = Builder.CreateICmpEQ(486 Kind, ConstantInt::get(Type::getInt16Ty(C),487 IsHIP ? object::OffloadKind::OFK_HIP488 : object::OffloadKind::OFK_Cuda));489 Builder.CreateCondBr(KindCond, IfKindBB, IfEndBB);490 Builder.SetInsertPoint(IfKindBB);491 auto *FnCond = Builder.CreateICmpEQ(492 Size, ConstantInt::getNullValue(Type::getInt64Ty(C)));493 Builder.CreateCondBr(FnCond, IfThenBB, IfElseBB);494 495 // Create kernel registration code.496 Builder.SetInsertPoint(IfThenBB);497 Builder.CreateCall(RegFunc, {RegGlobalsFn->arg_begin(), Addr, Name, Name,498 ConstantInt::get(Type::getInt32Ty(C), -1),499 ConstantPointerNull::get(Int8PtrTy),500 ConstantPointerNull::get(Int8PtrTy),501 ConstantPointerNull::get(Int8PtrTy),502 ConstantPointerNull::get(Int8PtrTy),503 ConstantPointerNull::get(Int32PtrTy)});504 Builder.CreateBr(IfEndBB);505 Builder.SetInsertPoint(IfElseBB);506 507 auto *Switch = Builder.CreateSwitch(Type, IfEndBB);508 // Create global variable registration code.509 Builder.SetInsertPoint(SwGlobalBB);510 Builder.CreateCall(RegVar,511 {RegGlobalsFn->arg_begin(), Addr, Name, Name, Extern, Size,512 Const, ConstantInt::get(Type::getInt32Ty(C), 0)});513 Builder.CreateBr(IfEndBB);514 Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalEntry),515 SwGlobalBB);516 517 // Create managed variable registration code.518 Builder.SetInsertPoint(SwManagedBB);519 Builder.CreateCall(RegManagedVar, {RegGlobalsFn->arg_begin(), AuxAddr, Addr,520 Name, Size, Data});521 Builder.CreateBr(IfEndBB);522 Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalManagedEntry),523 SwManagedBB);524 // Create surface variable registration code.525 Builder.SetInsertPoint(SwSurfaceBB);526 if (EmitSurfacesAndTextures)527 Builder.CreateCall(RegSurface, {RegGlobalsFn->arg_begin(), Addr, Name, Name,528 Data, Extern});529 Builder.CreateBr(IfEndBB);530 Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalSurfaceEntry),531 SwSurfaceBB);532 533 // Create texture variable registration code.534 Builder.SetInsertPoint(SwTextureBB);535 if (EmitSurfacesAndTextures)536 Builder.CreateCall(RegTexture, {RegGlobalsFn->arg_begin(), Addr, Name, Name,537 Data, Normalized, Extern});538 Builder.CreateBr(IfEndBB);539 Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalTextureEntry),540 SwTextureBB);541 542 Builder.SetInsertPoint(IfEndBB);543 auto *NewEntry = Builder.CreateInBoundsGEP(544 offloading::getEntryTy(M), Entry, ConstantInt::get(getSizeTTy(M), 1));545 auto *Cmp = Builder.CreateICmpEQ(546 NewEntry,547 ConstantExpr::getInBoundsGetElementPtr(548 ArrayType::get(offloading::getEntryTy(M), 0), EntriesE,549 ArrayRef<Constant *>({ConstantInt::get(getSizeTTy(M), 0),550 ConstantInt::get(getSizeTTy(M), 0)})));551 Entry->addIncoming(552 ConstantExpr::getInBoundsGetElementPtr(553 ArrayType::get(offloading::getEntryTy(M), 0), EntriesB,554 ArrayRef<Constant *>({ConstantInt::get(getSizeTTy(M), 0),555 ConstantInt::get(getSizeTTy(M), 0)})),556 &RegGlobalsFn->getEntryBlock());557 Entry->addIncoming(NewEntry, IfEndBB);558 Builder.CreateCondBr(Cmp, ExitBB, EntryBB);559 Builder.SetInsertPoint(ExitBB);560 Builder.CreateRetVoid();561 562 return RegGlobalsFn;563}564 565// Create the constructor and destructor to register the fatbinary with the CUDA566// runtime.567void createRegisterFatbinFunction(Module &M, GlobalVariable *FatbinDesc,568 bool IsHIP, EntryArrayTy EntryArray,569 StringRef Suffix,570 bool EmitSurfacesAndTextures) {571 LLVMContext &C = M.getContext();572 auto *CtorFuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);573 auto *CtorFunc = Function::Create(574 CtorFuncTy, GlobalValue::InternalLinkage,575 (IsHIP ? ".hip.fatbin_reg" : ".cuda.fatbin_reg") + Suffix, &M);576 CtorFunc->setSection(".text.startup");577 578 auto *DtorFuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);579 auto *DtorFunc = Function::Create(580 DtorFuncTy, GlobalValue::InternalLinkage,581 (IsHIP ? ".hip.fatbin_unreg" : ".cuda.fatbin_unreg") + Suffix, &M);582 DtorFunc->setSection(".text.startup");583 584 auto *PtrTy = PointerType::getUnqual(C);585 586 // Get the __cudaRegisterFatBinary function declaration.587 auto *RegFatTy = FunctionType::get(PtrTy, PtrTy, /*isVarArg=*/false);588 FunctionCallee RegFatbin = M.getOrInsertFunction(589 IsHIP ? "__hipRegisterFatBinary" : "__cudaRegisterFatBinary", RegFatTy);590 // Get the __cudaRegisterFatBinaryEnd function declaration.591 auto *RegFatEndTy =592 FunctionType::get(Type::getVoidTy(C), PtrTy, /*isVarArg=*/false);593 FunctionCallee RegFatbinEnd =594 M.getOrInsertFunction("__cudaRegisterFatBinaryEnd", RegFatEndTy);595 // Get the __cudaUnregisterFatBinary function declaration.596 auto *UnregFatTy =597 FunctionType::get(Type::getVoidTy(C), PtrTy, /*isVarArg=*/false);598 FunctionCallee UnregFatbin = M.getOrInsertFunction(599 IsHIP ? "__hipUnregisterFatBinary" : "__cudaUnregisterFatBinary",600 UnregFatTy);601 602 auto *AtExitTy =603 FunctionType::get(Type::getInt32Ty(C), PtrTy, /*isVarArg=*/false);604 FunctionCallee AtExit = M.getOrInsertFunction("atexit", AtExitTy);605 606 auto *BinaryHandleGlobal = new llvm::GlobalVariable(607 M, PtrTy, false, llvm::GlobalValue::InternalLinkage,608 llvm::ConstantPointerNull::get(PtrTy),609 (IsHIP ? ".hip.binary_handle" : ".cuda.binary_handle") + Suffix);610 611 // Create the constructor to register this image with the runtime.612 IRBuilder<> CtorBuilder(BasicBlock::Create(C, "entry", CtorFunc));613 CallInst *Handle = CtorBuilder.CreateCall(614 RegFatbin,615 ConstantExpr::getPointerBitCastOrAddrSpaceCast(FatbinDesc, PtrTy));616 CtorBuilder.CreateAlignedStore(617 Handle, BinaryHandleGlobal,618 Align(M.getDataLayout().getPointerTypeSize(PtrTy)));619 CtorBuilder.CreateCall(createRegisterGlobalsFunction(M, IsHIP, EntryArray,620 Suffix,621 EmitSurfacesAndTextures),622 Handle);623 if (!IsHIP)624 CtorBuilder.CreateCall(RegFatbinEnd, Handle);625 CtorBuilder.CreateCall(AtExit, DtorFunc);626 CtorBuilder.CreateRetVoid();627 628 // Create the destructor to unregister the image with the runtime. We cannot629 // use a standard global destructor after CUDA 9.2 so this must be called by630 // `atexit()` instead.631 IRBuilder<> DtorBuilder(BasicBlock::Create(C, "entry", DtorFunc));632 LoadInst *BinaryHandle = DtorBuilder.CreateAlignedLoad(633 PtrTy, BinaryHandleGlobal,634 Align(M.getDataLayout().getPointerTypeSize(PtrTy)));635 DtorBuilder.CreateCall(UnregFatbin, BinaryHandle);636 DtorBuilder.CreateRetVoid();637 638 // Add this function to constructors.639 appendToGlobalCtors(M, CtorFunc, /*Priority=*/101);640}641 642/// SYCLWrapper helper class that creates all LLVM IRs wrapping given images.643class SYCLWrapper {644public:645 SYCLWrapper(Module &M, const SYCLJITOptions &Options)646 : M(M), C(M.getContext()), Options(Options) {647 EntryTy = offloading::getEntryTy(M);648 SyclDeviceImageTy = getSyclDeviceImageTy();649 SyclBinDescTy = getSyclBinDescTy();650 }651 652 /// Creates binary descriptor for the given device images. Binary descriptor653 /// is an object that is passed to the offloading runtime at program startup654 /// and it describes all device images available in the executable or shared655 /// library. It is defined as follows:656 ///657 /// \code658 /// __attribute__((visibility("hidden")))659 /// __tgt_offload_entry *__sycl_offload_entries_arr0[];660 /// ...661 /// __attribute__((visibility("hidden")))662 /// __tgt_offload_entry *__sycl_offload_entries_arrN[];663 ///664 /// __attribute__((visibility("hidden")))665 /// extern const char *CompileOptions = "...";666 /// ...667 /// __attribute__((visibility("hidden")))668 /// extern const char *LinkOptions = "...";669 /// ...670 ///671 /// static const char Image0[] = { ... };672 /// ...673 /// static const char ImageN[] = { ... };674 ///675 /// static const __sycl.tgt_device_image Images[] = {676 /// {677 /// Version, // Version678 /// OffloadKind, // OffloadKind679 /// Format, // Format of the image.680 // TripleString, // Arch681 /// CompileOptions, // CompileOptions682 /// LinkOptions, // LinkOptions683 /// Image0, // ImageStart684 /// Image0 + IMAGE0_SIZE, // ImageEnd685 /// __sycl_offload_entries_arr0, // EntriesBegin686 /// __sycl_offload_entries_arr0 + ENTRIES0_SIZE, // EntriesEnd687 /// NULL, // PropertiesBegin688 /// NULL, // PropertiesEnd689 /// },690 /// ...691 /// };692 ///693 /// static const __sycl.tgt_bin_desc FatbinDesc = {694 /// Version, //Version695 /// sizeof(Images) / sizeof(Images[0]), //NumDeviceImages696 /// Images, //DeviceImages697 /// NULL, //HostEntriesBegin698 /// NULL //HostEntriesEnd699 /// };700 /// \endcode701 ///702 /// \returns Global variable that represents FatbinDesc.703 GlobalVariable *createFatbinDesc(ArrayRef<OffloadFile> OffloadFiles) {704 StringRef OffloadKindTag = ".sycl_offloading.";705 SmallVector<Constant *> WrappedImages;706 WrappedImages.reserve(OffloadFiles.size());707 for (size_t I = 0, E = OffloadFiles.size(); I != E; ++I)708 WrappedImages.push_back(709 wrapImage(*OffloadFiles[I].getBinary(), Twine(I), OffloadKindTag));710 711 return combineWrappedImages(WrappedImages, OffloadKindTag);712 }713 714 void createRegisterFatbinFunction(GlobalVariable *FatbinDesc) {715 FunctionType *FuncTy =716 FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);717 Function *Func = Function::Create(FuncTy, GlobalValue::InternalLinkage,718 Twine("sycl") + ".descriptor_reg", &M);719 Func->setSection(".text.startup");720 721 // Get RegFuncName function declaration.722 FunctionType *RegFuncTy =723 FunctionType::get(Type::getVoidTy(C), PointerType::getUnqual(C),724 /*isVarArg=*/false);725 FunctionCallee RegFuncC =726 M.getOrInsertFunction("__sycl_register_lib", RegFuncTy);727 728 // Construct function body.729 IRBuilder Builder(BasicBlock::Create(C, "entry", Func));730 Builder.CreateCall(RegFuncC, FatbinDesc);731 Builder.CreateRetVoid();732 733 // Add this function to constructors.734 appendToGlobalCtors(M, Func, /*Priority*/ 1);735 }736 737 void createUnregisterFunction(GlobalVariable *FatbinDesc) {738 FunctionType *FuncTy =739 FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);740 Function *Func = Function::Create(FuncTy, GlobalValue::InternalLinkage,741 "sycl.descriptor_unreg", &M);742 Func->setSection(".text.startup");743 744 // Get UnregFuncName function declaration.745 FunctionType *UnRegFuncTy =746 FunctionType::get(Type::getVoidTy(C), PointerType::getUnqual(C),747 /*isVarArg=*/false);748 FunctionCallee UnRegFuncC =749 M.getOrInsertFunction("__sycl_unregister_lib", UnRegFuncTy);750 751 // Construct function body752 IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));753 Builder.CreateCall(UnRegFuncC, FatbinDesc);754 Builder.CreateRetVoid();755 756 // Add this function to global destructors.757 appendToGlobalDtors(M, Func, /*Priority*/ 1);758 }759 760private:761 IntegerType *getSizeTTy() {762 switch (M.getDataLayout().getPointerSize()) {763 case 4:764 return Type::getInt32Ty(C);765 case 8:766 return Type::getInt64Ty(C);767 }768 llvm_unreachable("unsupported pointer type size");769 }770 771 SmallVector<Constant *, 2> getSizetConstPair(size_t First, size_t Second) {772 IntegerType *SizeTTy = getSizeTTy();773 return SmallVector<Constant *, 2>{ConstantInt::get(SizeTTy, First),774 ConstantInt::get(SizeTTy, Second)};775 }776 777 /// Note: Properties aren't supported and the support is going778 /// to be added later.779 /// Creates a structure corresponding to:780 /// SYCL specific image descriptor type.781 /// \code782 /// struct __sycl.tgt_device_image {783 /// // Version of this structure - for backward compatibility;784 /// // all modifications which change order/type/offsets of existing fields785 /// // should increment the version.786 /// uint16_t Version;787 /// // The kind of offload model the image employs.788 /// uint8_t OffloadKind;789 /// // Format of the image data - SPIRV, LLVMIR bitcode, etc.790 /// uint8_t Format;791 /// // Null-terminated string representation of the device's target792 /// // architecture.793 /// const char *Arch;794 /// // A null-terminated string; target- and compiler-specific options795 /// // which are passed to the device compiler at runtime.796 /// const char *CompileOptions;797 /// // A null-terminated string; target- and compiler-specific options798 /// // which are passed to the device linker at runtime.799 /// const char *LinkOptions;800 /// // Pointer to the device binary image start.801 /// void *ImageStart;802 /// // Pointer to the device binary image end.803 /// void *ImageEnd;804 /// // The entry table.805 /// __tgt_offload_entry *EntriesBegin;806 /// __tgt_offload_entry *EntriesEnd;807 /// const char *PropertiesBegin;808 /// const char *PropertiesEnd;809 /// };810 /// \endcode811 StructType *getSyclDeviceImageTy() {812 return StructType::create(813 {814 Type::getInt16Ty(C), // Version815 Type::getInt8Ty(C), // OffloadKind816 Type::getInt8Ty(C), // Format817 PointerType::getUnqual(C), // Arch818 PointerType::getUnqual(C), // CompileOptions819 PointerType::getUnqual(C), // LinkOptions820 PointerType::getUnqual(C), // ImageStart821 PointerType::getUnqual(C), // ImageEnd822 PointerType::getUnqual(C), // EntriesBegin823 PointerType::getUnqual(C), // EntriesEnd824 PointerType::getUnqual(C), // PropertiesBegin825 PointerType::getUnqual(C) // PropertiesEnd826 },827 "__sycl.tgt_device_image");828 }829 830 /// Creates a structure for SYCL specific binary descriptor type. Corresponds831 /// to:832 ///833 /// \code834 /// struct __sycl.tgt_bin_desc {835 /// // version of this structure - for backward compatibility;836 /// // all modifications which change order/type/offsets of existing fields837 /// // should increment the version.838 /// uint16_t Version;839 /// uint16_t NumDeviceImages;840 /// __sycl.tgt_device_image *DeviceImages;841 /// // the offload entry table842 /// __tgt_offload_entry *HostEntriesBegin;843 /// __tgt_offload_entry *HostEntriesEnd;844 /// };845 /// \endcode846 StructType *getSyclBinDescTy() {847 return StructType::create(848 {Type::getInt16Ty(C), Type::getInt16Ty(C), PointerType::getUnqual(C),849 PointerType::getUnqual(C), PointerType::getUnqual(C)},850 "__sycl.tgt_bin_desc");851 }852 853 /// Adds a global readonly variable that is initialized by given854 /// \p Initializer to the module.855 GlobalVariable *addGlobalArrayVariable(const Twine &Name,856 ArrayRef<char> Initializer,857 const Twine &Section = "") {858 Constant *Arr = ConstantDataArray::get(M.getContext(), Initializer);859 GlobalVariable *Var =860 new GlobalVariable(M, Arr->getType(), /*isConstant*/ true,861 GlobalVariable::InternalLinkage, Arr, Name);862 Var->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);863 864 SmallVector<char, 32> NameBuf;865 StringRef SectionName = Section.toStringRef(NameBuf);866 if (!SectionName.empty())867 Var->setSection(SectionName);868 return Var;869 }870 871 /// Adds given \p Buf as a global variable into the module.872 /// \returns Pair of pointers that point at the beginning and the end of the873 /// variable.874 std::pair<Constant *, Constant *>875 addArrayToModule(ArrayRef<char> Buf, const Twine &Name,876 const Twine &Section = "") {877 GlobalVariable *Var = addGlobalArrayVariable(Name, Buf, Section);878 Constant *ImageB = ConstantExpr::getGetElementPtr(Var->getValueType(), Var,879 getSizetConstPair(0, 0));880 Constant *ImageE = ConstantExpr::getGetElementPtr(881 Var->getValueType(), Var, getSizetConstPair(0, Buf.size()));882 return std::make_pair(ImageB, ImageE);883 }884 885 /// Adds given \p Data as constant byte array in the module.886 /// \returns Constant pointer to the added data. The pointer type does not887 /// carry size information.888 Constant *addRawDataToModule(ArrayRef<char> Data, const Twine &Name) {889 GlobalVariable *Var = addGlobalArrayVariable(Name, Data);890 Constant *DataPtr = ConstantExpr::getGetElementPtr(Var->getValueType(), Var,891 getSizetConstPair(0, 0));892 return DataPtr;893 }894 895 /// Creates a global variable of const char* type and creates an896 /// initializer that initializes it with \p Str.897 ///898 /// \returns Link-time constant pointer (constant expr) to that899 /// variable.900 Constant *addStringToModule(StringRef Str, const Twine &Name) {901 Constant *Arr = ConstantDataArray::getString(C, Str);902 GlobalVariable *Var =903 new GlobalVariable(M, Arr->getType(), /*isConstant*/ true,904 GlobalVariable::InternalLinkage, Arr, Name);905 Var->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);906 ConstantInt *Zero = ConstantInt::get(getSizeTTy(), 0);907 Constant *ZeroZero[] = {Zero, Zero};908 return ConstantExpr::getGetElementPtr(Var->getValueType(), Var, ZeroZero);909 }910 911 /// Each image contains its own set of symbols, which may contain different912 /// symbols than other images. This function constructs an array of913 /// symbol entries for a particular image.914 ///915 /// \returns Pointers to the beginning and end of the array.916 std::pair<Constant *, Constant *>917 initOffloadEntriesPerImage(StringRef Entries, const Twine &OffloadKindTag) {918 SmallVector<Constant *> EntriesInits;919 std::unique_ptr<MemoryBuffer> MB = MemoryBuffer::getMemBuffer(920 Entries, /*BufferName*/ "", /*RequiresNullTerminator*/ false);921 for (line_iterator LI(*MB); !LI.is_at_eof(); ++LI) {922 GlobalVariable *GV =923 emitOffloadingEntry(M, /*Kind*/ OffloadKind::OFK_SYCL,924 Constant::getNullValue(PointerType::getUnqual(C)),925 /*Name*/ *LI, /*Size*/ 0,926 /*Flags*/ 0, /*Data*/ 0);927 EntriesInits.push_back(GV->getInitializer());928 }929 930 Constant *Arr = ConstantArray::get(931 ArrayType::get(EntryTy, EntriesInits.size()), EntriesInits);932 GlobalVariable *EntriesGV = new GlobalVariable(933 M, Arr->getType(), /*isConstant*/ true, GlobalVariable::InternalLinkage,934 Arr, OffloadKindTag + "entries_arr");935 936 Constant *EntriesB = ConstantExpr::getGetElementPtr(937 EntriesGV->getValueType(), EntriesGV, getSizetConstPair(0, 0));938 Constant *EntriesE = ConstantExpr::getGetElementPtr(939 EntriesGV->getValueType(), EntriesGV,940 getSizetConstPair(0, EntriesInits.size()));941 return std::make_pair(EntriesB, EntriesE);942 }943 944 Constant *wrapImage(const OffloadBinary &OB, const Twine &ImageID,945 StringRef OffloadKindTag) {946 // Note: Intel DPC++ compiler had 2 versions of this structure947 // and clang++ has a third different structure. To avoid ABI incompatibility948 // between generated device images the Version here starts from 3.949 constexpr uint16_t DeviceImageStructVersion = 3;950 Constant *Version =951 ConstantInt::get(Type::getInt16Ty(C), DeviceImageStructVersion);952 Constant *OffloadKindConstant = ConstantInt::get(953 Type::getInt8Ty(C), static_cast<uint8_t>(OB.getOffloadKind()));954 Constant *ImageKindConstant = ConstantInt::get(955 Type::getInt8Ty(C), static_cast<uint8_t>(OB.getImageKind()));956 StringRef Triple = OB.getString("triple");957 Constant *TripleConstant =958 addStringToModule(Triple, Twine(OffloadKindTag) + "target." + ImageID);959 Constant *CompileOptions =960 addStringToModule(Options.CompileOptions,961 Twine(OffloadKindTag) + "opts.compile." + ImageID);962 Constant *LinkOptions = addStringToModule(963 Options.LinkOptions, Twine(OffloadKindTag) + "opts.link." + ImageID);964 965 // Note: NULL for now.966 std::pair<Constant *, Constant *> PropertiesConstants = {967 Constant::getNullValue(PointerType::getUnqual(C)),968 Constant::getNullValue(PointerType::getUnqual(C))};969 970 StringRef RawImage = OB.getImage();971 std::pair<Constant *, Constant *> Binary = addArrayToModule(972 ArrayRef<char>(RawImage.begin(), RawImage.end()),973 Twine(OffloadKindTag) + ImageID + ".data", ".llvm.offloading");974 975 // For SYCL images offload entries are defined here per image.976 std::pair<Constant *, Constant *> ImageEntriesPtrs =977 initOffloadEntriesPerImage(OB.getString("symbols"), OffloadKindTag);978 979 // .first and .second arguments below correspond to start and end pointers980 // respectively.981 Constant *WrappedBinary = ConstantStruct::get(982 SyclDeviceImageTy, Version, OffloadKindConstant, ImageKindConstant,983 TripleConstant, CompileOptions, LinkOptions, Binary.first,984 Binary.second, ImageEntriesPtrs.first, ImageEntriesPtrs.second,985 PropertiesConstants.first, PropertiesConstants.second);986 987 return WrappedBinary;988 }989 990 GlobalVariable *combineWrappedImages(ArrayRef<Constant *> WrappedImages,991 StringRef OffloadKindTag) {992 Constant *ImagesData = ConstantArray::get(993 ArrayType::get(SyclDeviceImageTy, WrappedImages.size()), WrappedImages);994 GlobalVariable *ImagesGV =995 new GlobalVariable(M, ImagesData->getType(), /*isConstant*/ true,996 GlobalValue::InternalLinkage, ImagesData,997 Twine(OffloadKindTag) + "device_images");998 ImagesGV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);999 1000 ConstantInt *Zero = ConstantInt::get(getSizeTTy(), 0);1001 Constant *ZeroZero[] = {Zero, Zero};1002 Constant *ImagesB = ConstantExpr::getGetElementPtr(ImagesGV->getValueType(),1003 ImagesGV, ZeroZero);1004 1005 Constant *EntriesB = Constant::getNullValue(PointerType::getUnqual(C));1006 Constant *EntriesE = Constant::getNullValue(PointerType::getUnqual(C));1007 static constexpr uint16_t BinDescStructVersion = 1;1008 Constant *DescInit = ConstantStruct::get(1009 SyclBinDescTy,1010 ConstantInt::get(Type::getInt16Ty(C), BinDescStructVersion),1011 ConstantInt::get(Type::getInt16Ty(C), WrappedImages.size()), ImagesB,1012 EntriesB, EntriesE);1013 1014 return new GlobalVariable(M, DescInit->getType(), /*isConstant*/ true,1015 GlobalValue::InternalLinkage, DescInit,1016 Twine(OffloadKindTag) + "descriptor");1017 }1018 1019 Module &M;1020 LLVMContext &C;1021 SYCLJITOptions Options;1022 1023 StructType *EntryTy = nullptr;1024 StructType *SyclDeviceImageTy = nullptr;1025 StructType *SyclBinDescTy = nullptr;1026}; // end of SYCLWrapper1027 1028} // namespace1029 1030Error offloading::wrapOpenMPBinaries(Module &M, ArrayRef<ArrayRef<char>> Images,1031 EntryArrayTy EntryArray,1032 llvm::StringRef Suffix, bool Relocatable) {1033 GlobalVariable *Desc =1034 createBinDesc(M, Images, EntryArray, Suffix, Relocatable);1035 if (!Desc)1036 return createStringError(inconvertibleErrorCode(),1037 "No binary descriptors created.");1038 createRegisterFunction(M, Desc, Suffix);1039 return Error::success();1040}1041 1042Error offloading::wrapCudaBinary(Module &M, ArrayRef<char> Image,1043 EntryArrayTy EntryArray,1044 llvm::StringRef Suffix,1045 bool EmitSurfacesAndTextures) {1046 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/false, Suffix);1047 if (!Desc)1048 return createStringError(inconvertibleErrorCode(),1049 "No fatbin section created.");1050 1051 createRegisterFatbinFunction(M, Desc, /*IsHip=*/false, EntryArray, Suffix,1052 EmitSurfacesAndTextures);1053 return Error::success();1054}1055 1056Error offloading::wrapHIPBinary(Module &M, ArrayRef<char> Image,1057 EntryArrayTy EntryArray, llvm::StringRef Suffix,1058 bool EmitSurfacesAndTextures) {1059 GlobalVariable *Desc = createFatbinDesc(M, Image, /*IsHip=*/true, Suffix);1060 if (!Desc)1061 return createStringError(inconvertibleErrorCode(),1062 "No fatbin section created.");1063 1064 createRegisterFatbinFunction(M, Desc, /*IsHip=*/true, EntryArray, Suffix,1065 EmitSurfacesAndTextures);1066 return Error::success();1067}1068 1069Error llvm::offloading::wrapSYCLBinaries(llvm::Module &M, ArrayRef<char> Buffer,1070 SYCLJITOptions Options) {1071 SYCLWrapper W(M, Options);1072 MemoryBufferRef MBR(StringRef(Buffer.begin(), Buffer.size()),1073 /*Identifier*/ "");1074 SmallVector<OffloadFile> OffloadFiles;1075 if (Error E = extractOffloadBinaries(MBR, OffloadFiles))1076 return E;1077 1078 GlobalVariable *Desc = W.createFatbinDesc(OffloadFiles);1079 if (!Desc)1080 return createStringError(inconvertibleErrorCode(),1081 "No binary descriptors created.");1082 1083 W.createRegisterFatbinFunction(Desc);1084 W.createUnregisterFunction(Desc);1085 return Error::success();1086}1087