1524 lines · cpp
1//===----- CGHLSLRuntime.cpp - Interface to HLSL Runtimes -----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This provides an abstract class for HLSL code generation. Concrete10// subclasses of this implement code generation for specific HLSL11// runtime libraries.12//13//===----------------------------------------------------------------------===//14 15#include "CGHLSLRuntime.h"16#include "CGDebugInfo.h"17#include "CGRecordLayout.h"18#include "CodeGenFunction.h"19#include "CodeGenModule.h"20#include "HLSLBufferLayoutBuilder.h"21#include "TargetInfo.h"22#include "clang/AST/ASTContext.h"23#include "clang/AST/Attrs.inc"24#include "clang/AST/Decl.h"25#include "clang/AST/HLSLResource.h"26#include "clang/AST/RecursiveASTVisitor.h"27#include "clang/AST/Type.h"28#include "clang/Basic/DiagnosticFrontend.h"29#include "clang/Basic/TargetOptions.h"30#include "llvm/ADT/DenseMap.h"31#include "llvm/ADT/ScopeExit.h"32#include "llvm/ADT/SmallString.h"33#include "llvm/ADT/SmallVector.h"34#include "llvm/Frontend/HLSL/RootSignatureMetadata.h"35#include "llvm/IR/Constants.h"36#include "llvm/IR/DerivedTypes.h"37#include "llvm/IR/GlobalVariable.h"38#include "llvm/IR/LLVMContext.h"39#include "llvm/IR/Metadata.h"40#include "llvm/IR/Module.h"41#include "llvm/IR/Type.h"42#include "llvm/IR/Value.h"43#include "llvm/Support/Alignment.h"44#include "llvm/Support/ErrorHandling.h"45#include "llvm/Support/FormatVariadic.h"46#include <cstdint>47#include <optional>48 49using namespace clang;50using namespace CodeGen;51using namespace clang::hlsl;52using namespace llvm;53 54using llvm::hlsl::CBufferRowSizeInBytes;55 56namespace {57 58void addDxilValVersion(StringRef ValVersionStr, llvm::Module &M) {59 // The validation of ValVersionStr is done at HLSLToolChain::TranslateArgs.60 // Assume ValVersionStr is legal here.61 VersionTuple Version;62 if (Version.tryParse(ValVersionStr) || Version.getBuild() ||63 Version.getSubminor() || !Version.getMinor()) {64 return;65 }66 67 uint64_t Major = Version.getMajor();68 uint64_t Minor = *Version.getMinor();69 70 auto &Ctx = M.getContext();71 IRBuilder<> B(M.getContext());72 MDNode *Val = MDNode::get(Ctx, {ConstantAsMetadata::get(B.getInt32(Major)),73 ConstantAsMetadata::get(B.getInt32(Minor))});74 StringRef DXILValKey = "dx.valver";75 auto *DXILValMD = M.getOrInsertNamedMetadata(DXILValKey);76 DXILValMD->addOperand(Val);77}78 79void addRootSignatureMD(llvm::dxbc::RootSignatureVersion RootSigVer,80 ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,81 llvm::Function *Fn, llvm::Module &M) {82 auto &Ctx = M.getContext();83 84 llvm::hlsl::rootsig::MetadataBuilder RSBuilder(Ctx, Elements);85 MDNode *RootSignature = RSBuilder.BuildRootSignature();86 87 ConstantAsMetadata *Version = ConstantAsMetadata::get(ConstantInt::get(88 llvm::Type::getInt32Ty(Ctx), llvm::to_underlying(RootSigVer)));89 ValueAsMetadata *EntryFunc = Fn ? ValueAsMetadata::get(Fn) : nullptr;90 MDNode *MDVals = MDNode::get(Ctx, {EntryFunc, RootSignature, Version});91 92 StringRef RootSignatureValKey = "dx.rootsignatures";93 auto *RootSignatureValMD = M.getOrInsertNamedMetadata(RootSignatureValKey);94 RootSignatureValMD->addOperand(MDVals);95}96 97// Find array variable declaration from nested array subscript AST nodes98static const ValueDecl *getArrayDecl(const ArraySubscriptExpr *ASE) {99 const Expr *E = nullptr;100 while (ASE != nullptr) {101 E = ASE->getBase()->IgnoreImpCasts();102 if (!E)103 return nullptr;104 ASE = dyn_cast<ArraySubscriptExpr>(E);105 }106 if (const DeclRefExpr *DRE = dyn_cast_or_null<DeclRefExpr>(E))107 return DRE->getDecl();108 return nullptr;109}110 111// Get the total size of the array, or -1 if the array is unbounded.112static int getTotalArraySize(ASTContext &AST, const clang::Type *Ty) {113 Ty = Ty->getUnqualifiedDesugaredType();114 assert(Ty->isArrayType() && "expected array type");115 if (Ty->isIncompleteArrayType())116 return -1;117 return AST.getConstantArrayElementCount(cast<ConstantArrayType>(Ty));118}119 120static Value *buildNameForResource(llvm::StringRef BaseName,121 CodeGenModule &CGM) {122 llvm::SmallString<64> GlobalName = {BaseName, ".str"};123 return CGM.GetAddrOfConstantCString(BaseName.str(), GlobalName.c_str())124 .getPointer();125}126 127static CXXMethodDecl *lookupMethod(CXXRecordDecl *Record, StringRef Name,128 StorageClass SC = SC_None) {129 for (auto *Method : Record->methods()) {130 if (Method->getStorageClass() == SC && Method->getName() == Name)131 return Method;132 }133 return nullptr;134}135 136static CXXMethodDecl *lookupResourceInitMethodAndSetupArgs(137 CodeGenModule &CGM, CXXRecordDecl *ResourceDecl, llvm::Value *Range,138 llvm::Value *Index, StringRef Name, ResourceBindingAttrs &Binding,139 CallArgList &Args) {140 assert(Binding.hasBinding() && "at least one binding attribute expected");141 142 ASTContext &AST = CGM.getContext();143 CXXMethodDecl *CreateMethod = nullptr;144 Value *NameStr = buildNameForResource(Name, CGM);145 Value *Space = llvm::ConstantInt::get(CGM.IntTy, Binding.getSpace());146 147 if (Binding.isExplicit()) {148 // explicit binding149 auto *RegSlot = llvm::ConstantInt::get(CGM.IntTy, Binding.getSlot());150 Args.add(RValue::get(RegSlot), AST.UnsignedIntTy);151 const char *Name = Binding.hasCounterImplicitOrderID()152 ? "__createFromBindingWithImplicitCounter"153 : "__createFromBinding";154 CreateMethod = lookupMethod(ResourceDecl, Name, SC_Static);155 } else {156 // implicit binding157 auto *OrderID =158 llvm::ConstantInt::get(CGM.IntTy, Binding.getImplicitOrderID());159 Args.add(RValue::get(OrderID), AST.UnsignedIntTy);160 const char *Name = Binding.hasCounterImplicitOrderID()161 ? "__createFromImplicitBindingWithImplicitCounter"162 : "__createFromImplicitBinding";163 CreateMethod = lookupMethod(ResourceDecl, Name, SC_Static);164 }165 Args.add(RValue::get(Space), AST.UnsignedIntTy);166 Args.add(RValue::get(Range), AST.IntTy);167 Args.add(RValue::get(Index), AST.UnsignedIntTy);168 Args.add(RValue::get(NameStr), AST.getPointerType(AST.CharTy.withConst()));169 if (Binding.hasCounterImplicitOrderID()) {170 uint32_t CounterBinding = Binding.getCounterImplicitOrderID();171 auto *CounterOrderID = llvm::ConstantInt::get(CGM.IntTy, CounterBinding);172 Args.add(RValue::get(CounterOrderID), AST.UnsignedIntTy);173 }174 175 return CreateMethod;176}177 178static void callResourceInitMethod(CodeGenFunction &CGF,179 CXXMethodDecl *CreateMethod,180 CallArgList &Args, Address ReturnAddress) {181 llvm::Constant *CalleeFn = CGF.CGM.GetAddrOfFunction(CreateMethod);182 const FunctionProtoType *Proto =183 CreateMethod->getType()->getAs<FunctionProtoType>();184 const CGFunctionInfo &FnInfo =185 CGF.CGM.getTypes().arrangeFreeFunctionCall(Args, Proto, false);186 ReturnValueSlot ReturnValue(ReturnAddress, false);187 CGCallee Callee(CGCalleeInfo(Proto), CalleeFn);188 CGF.EmitCall(FnInfo, Callee, ReturnValue, Args, nullptr);189}190 191// Initializes local resource array variable. For multi-dimensional arrays it192// calls itself recursively to initialize its sub-arrays. The Index used in the193// resource constructor calls will begin at StartIndex and will be incremented194// for each array element. The last used resource Index is returned to the195// caller. If the function returns std::nullopt, it indicates an error.196static std::optional<llvm::Value *> initializeLocalResourceArray(197 CodeGenFunction &CGF, CXXRecordDecl *ResourceDecl,198 const ConstantArrayType *ArrayTy, AggValueSlot &ValueSlot,199 llvm::Value *Range, llvm::Value *StartIndex, StringRef ResourceName,200 ResourceBindingAttrs &Binding, ArrayRef<llvm::Value *> PrevGEPIndices,201 SourceLocation ArraySubsExprLoc) {202 203 ASTContext &AST = CGF.getContext();204 llvm::IntegerType *IntTy = CGF.CGM.IntTy;205 llvm::Value *Index = StartIndex;206 llvm::Value *One = llvm::ConstantInt::get(IntTy, 1);207 const uint64_t ArraySize = ArrayTy->getSExtSize();208 QualType ElemType = ArrayTy->getElementType();209 Address TmpArrayAddr = ValueSlot.getAddress();210 211 // Add additional index to the getelementptr call indices.212 // This index will be updated for each array element in the loops below.213 SmallVector<llvm::Value *> GEPIndices(PrevGEPIndices);214 GEPIndices.push_back(llvm::ConstantInt::get(IntTy, 0));215 216 // For array of arrays, recursively initialize the sub-arrays.217 if (ElemType->isArrayType()) {218 const ConstantArrayType *SubArrayTy = cast<ConstantArrayType>(ElemType);219 for (uint64_t I = 0; I < ArraySize; I++) {220 if (I > 0) {221 Index = CGF.Builder.CreateAdd(Index, One);222 GEPIndices.back() = llvm::ConstantInt::get(IntTy, I);223 }224 std::optional<llvm::Value *> MaybeIndex = initializeLocalResourceArray(225 CGF, ResourceDecl, SubArrayTy, ValueSlot, Range, Index, ResourceName,226 Binding, GEPIndices, ArraySubsExprLoc);227 if (!MaybeIndex)228 return std::nullopt;229 Index = *MaybeIndex;230 }231 return Index;232 }233 234 // For array of resources, initialize each resource in the array.235 llvm::Type *Ty = CGF.ConvertTypeForMem(ElemType);236 CharUnits ElemSize = AST.getTypeSizeInChars(ElemType);237 CharUnits Align =238 TmpArrayAddr.getAlignment().alignmentOfArrayElement(ElemSize);239 240 for (uint64_t I = 0; I < ArraySize; I++) {241 if (I > 0) {242 Index = CGF.Builder.CreateAdd(Index, One);243 GEPIndices.back() = llvm::ConstantInt::get(IntTy, I);244 }245 Address ReturnAddress =246 CGF.Builder.CreateGEP(TmpArrayAddr, GEPIndices, Ty, Align);247 248 CallArgList Args;249 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(250 CGF.CGM, ResourceDecl, Range, Index, ResourceName, Binding, Args);251 252 if (!CreateMethod)253 // This can happen if someone creates an array of structs that looks like254 // an HLSL resource record array but it does not have the required static255 // create method. No binding will be generated for it.256 return std::nullopt;257 258 callResourceInitMethod(CGF, CreateMethod, Args, ReturnAddress);259 }260 return Index;261}262 263} // namespace264 265llvm::Type *266CGHLSLRuntime::convertHLSLSpecificType(const Type *T,267 const CGHLSLOffsetInfo &OffsetInfo) {268 assert(T->isHLSLSpecificType() && "Not an HLSL specific type!");269 270 // Check if the target has a specific translation for this type first.271 if (llvm::Type *TargetTy =272 CGM.getTargetCodeGenInfo().getHLSLType(CGM, T, OffsetInfo))273 return TargetTy;274 275 llvm_unreachable("Generic handling of HLSL types is not supported.");276}277 278llvm::Triple::ArchType CGHLSLRuntime::getArch() {279 return CGM.getTarget().getTriple().getArch();280}281 282// Emits constant global variables for buffer constants declarations283// and creates metadata linking the constant globals with the buffer global.284void CGHLSLRuntime::emitBufferGlobalsAndMetadata(285 const HLSLBufferDecl *BufDecl, llvm::GlobalVariable *BufGV,286 const CGHLSLOffsetInfo &OffsetInfo) {287 LLVMContext &Ctx = CGM.getLLVMContext();288 289 // get the layout struct from constant buffer target type290 llvm::Type *BufType = BufGV->getValueType();291 llvm::StructType *LayoutStruct = cast<llvm::StructType>(292 cast<llvm::TargetExtType>(BufType)->getTypeParameter(0));293 294 SmallVector<std::pair<VarDecl *, uint32_t>> DeclsWithOffset;295 size_t OffsetIdx = 0;296 for (Decl *D : BufDecl->buffer_decls()) {297 if (isa<CXXRecordDecl, EmptyDecl>(D))298 // Nothing to do for this declaration.299 continue;300 if (isa<FunctionDecl>(D)) {301 // A function within an cbuffer is effectively a top-level function.302 CGM.EmitTopLevelDecl(D);303 continue;304 }305 VarDecl *VD = dyn_cast<VarDecl>(D);306 if (!VD)307 continue;308 309 QualType VDTy = VD->getType();310 if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {311 if (VD->getStorageClass() == SC_Static ||312 VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||313 VDTy->isHLSLResourceRecord() || VDTy->isHLSLResourceRecordArray()) {314 // Emit static and groupshared variables and resource classes inside315 // cbuffer as regular globals316 CGM.EmitGlobal(VD);317 } else {318 // Anything else that is not in the hlsl_constant address space must be319 // an empty struct or a zero-sized array and can be ignored320 assert(BufDecl->getASTContext().getTypeSize(VDTy) == 0 &&321 "constant buffer decl with non-zero sized type outside of "322 "hlsl_constant address space");323 }324 continue;325 }326 327 DeclsWithOffset.emplace_back(VD, OffsetInfo[OffsetIdx++]);328 }329 330 if (!OffsetInfo.empty())331 llvm::stable_sort(DeclsWithOffset, [](const auto &LHS, const auto &RHS) {332 return CGHLSLOffsetInfo::compareOffsets(LHS.second, RHS.second);333 });334 335 // Associate the buffer global variable with its constants336 SmallVector<llvm::Metadata *> BufGlobals;337 BufGlobals.reserve(DeclsWithOffset.size() + 1);338 BufGlobals.push_back(ValueAsMetadata::get(BufGV));339 340 auto ElemIt = LayoutStruct->element_begin();341 for (auto &[VD, _] : DeclsWithOffset) {342 if (CGM.getTargetCodeGenInfo().isHLSLPadding(*ElemIt))343 ++ElemIt;344 345 assert(ElemIt != LayoutStruct->element_end() &&346 "number of elements in layout struct does not match");347 llvm::Type *LayoutType = *ElemIt++;348 349 GlobalVariable *ElemGV =350 cast<GlobalVariable>(CGM.GetAddrOfGlobalVar(VD, LayoutType));351 BufGlobals.push_back(ValueAsMetadata::get(ElemGV));352 }353 assert(ElemIt == LayoutStruct->element_end() &&354 "number of elements in layout struct does not match");355 356 // add buffer metadata to the module357 CGM.getModule()358 .getOrInsertNamedMetadata("hlsl.cbs")359 ->addOperand(MDNode::get(Ctx, BufGlobals));360}361 362// Creates resource handle type for the HLSL buffer declaration363static const clang::HLSLAttributedResourceType *364createBufferHandleType(const HLSLBufferDecl *BufDecl) {365 ASTContext &AST = BufDecl->getASTContext();366 QualType QT = AST.getHLSLAttributedResourceType(367 AST.HLSLResourceTy, AST.getCanonicalTagType(BufDecl->getLayoutStruct()),368 HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));369 return cast<HLSLAttributedResourceType>(QT.getTypePtr());370}371 372CGHLSLOffsetInfo CGHLSLOffsetInfo::fromDecl(const HLSLBufferDecl &BufDecl) {373 CGHLSLOffsetInfo Result;374 375 // If we don't have packoffset info, just return an empty result.376 if (!BufDecl.hasValidPackoffset())377 return Result;378 379 for (Decl *D : BufDecl.buffer_decls()) {380 if (isa<CXXRecordDecl, EmptyDecl>(D) || isa<FunctionDecl>(D)) {381 continue;382 }383 VarDecl *VD = dyn_cast<VarDecl>(D);384 if (!VD || VD->getType().getAddressSpace() != LangAS::hlsl_constant)385 continue;386 387 if (!VD->hasAttrs()) {388 Result.Offsets.push_back(Unspecified);389 continue;390 }391 392 uint32_t Offset = Unspecified;393 for (auto *Attr : VD->getAttrs()) {394 if (auto *POA = dyn_cast<HLSLPackOffsetAttr>(Attr)) {395 Offset = POA->getOffsetInBytes();396 break;397 }398 auto *RBA = dyn_cast<HLSLResourceBindingAttr>(Attr);399 if (RBA &&400 RBA->getRegisterType() == HLSLResourceBindingAttr::RegisterType::C) {401 Offset = RBA->getSlotNumber() * CBufferRowSizeInBytes;402 break;403 }404 }405 Result.Offsets.push_back(Offset);406 }407 return Result;408}409 410// Codegen for HLSLBufferDecl411void CGHLSLRuntime::addBuffer(const HLSLBufferDecl *BufDecl) {412 413 assert(BufDecl->isCBuffer() && "tbuffer codegen is not supported yet");414 415 // create resource handle type for the buffer416 const clang::HLSLAttributedResourceType *ResHandleTy =417 createBufferHandleType(BufDecl);418 419 // empty constant buffer is ignored420 if (ResHandleTy->getContainedType()->getAsCXXRecordDecl()->isEmpty())421 return;422 423 // create global variable for the constant buffer424 CGHLSLOffsetInfo OffsetInfo = CGHLSLOffsetInfo::fromDecl(*BufDecl);425 llvm::Type *LayoutTy = convertHLSLSpecificType(ResHandleTy, OffsetInfo);426 llvm::GlobalVariable *BufGV = new GlobalVariable(427 LayoutTy, /*isConstant*/ false,428 GlobalValue::LinkageTypes::ExternalLinkage, PoisonValue::get(LayoutTy),429 llvm::formatv("{0}{1}", BufDecl->getName(),430 BufDecl->isCBuffer() ? ".cb" : ".tb"),431 GlobalValue::NotThreadLocal);432 CGM.getModule().insertGlobalVariable(BufGV);433 434 // Add globals for constant buffer elements and create metadata nodes435 emitBufferGlobalsAndMetadata(BufDecl, BufGV, OffsetInfo);436 437 // Initialize cbuffer from binding (implicit or explicit)438 initializeBufferFromBinding(BufDecl, BufGV);439}440 441void CGHLSLRuntime::addRootSignature(442 const HLSLRootSignatureDecl *SignatureDecl) {443 llvm::Module &M = CGM.getModule();444 Triple T(M.getTargetTriple());445 446 // Generated later with the function decl if not targeting root signature447 if (T.getEnvironment() != Triple::EnvironmentType::RootSignature)448 return;449 450 addRootSignatureMD(SignatureDecl->getVersion(),451 SignatureDecl->getRootElements(), nullptr, M);452}453 454llvm::StructType *455CGHLSLRuntime::getHLSLBufferLayoutType(const RecordType *StructType) {456 const auto Entry = LayoutTypes.find(StructType);457 if (Entry != LayoutTypes.end())458 return Entry->getSecond();459 return nullptr;460}461 462void CGHLSLRuntime::addHLSLBufferLayoutType(const RecordType *StructType,463 llvm::StructType *LayoutTy) {464 assert(getHLSLBufferLayoutType(StructType) == nullptr &&465 "layout type for this struct already exist");466 LayoutTypes[StructType] = LayoutTy;467}468 469void CGHLSLRuntime::finishCodeGen() {470 auto &TargetOpts = CGM.getTarget().getTargetOpts();471 auto &CodeGenOpts = CGM.getCodeGenOpts();472 auto &LangOpts = CGM.getLangOpts();473 llvm::Module &M = CGM.getModule();474 Triple T(M.getTargetTriple());475 if (T.getArch() == Triple::ArchType::dxil)476 addDxilValVersion(TargetOpts.DxilValidatorVersion, M);477 if (CodeGenOpts.ResMayAlias)478 M.setModuleFlag(llvm::Module::ModFlagBehavior::Error, "dx.resmayalias", 1);479 480 // NativeHalfType corresponds to the -fnative-half-type clang option which is481 // aliased by clang-dxc's -enable-16bit-types option. This option is used to482 // set the UseNativeLowPrecision DXIL module flag in the DirectX backend483 if (LangOpts.NativeHalfType)484 M.setModuleFlag(llvm::Module::ModFlagBehavior::Error, "dx.nativelowprec",485 1);486 487 generateGlobalCtorDtorCalls();488}489 490void clang::CodeGen::CGHLSLRuntime::setHLSLEntryAttributes(491 const FunctionDecl *FD, llvm::Function *Fn) {492 const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();493 assert(ShaderAttr && "All entry functions must have a HLSLShaderAttr");494 const StringRef ShaderAttrKindStr = "hlsl.shader";495 Fn->addFnAttr(ShaderAttrKindStr,496 llvm::Triple::getEnvironmentTypeName(ShaderAttr->getType()));497 if (HLSLNumThreadsAttr *NumThreadsAttr = FD->getAttr<HLSLNumThreadsAttr>()) {498 const StringRef NumThreadsKindStr = "hlsl.numthreads";499 std::string NumThreadsStr =500 formatv("{0},{1},{2}", NumThreadsAttr->getX(), NumThreadsAttr->getY(),501 NumThreadsAttr->getZ());502 Fn->addFnAttr(NumThreadsKindStr, NumThreadsStr);503 }504 if (HLSLWaveSizeAttr *WaveSizeAttr = FD->getAttr<HLSLWaveSizeAttr>()) {505 const StringRef WaveSizeKindStr = "hlsl.wavesize";506 std::string WaveSizeStr =507 formatv("{0},{1},{2}", WaveSizeAttr->getMin(), WaveSizeAttr->getMax(),508 WaveSizeAttr->getPreferred());509 Fn->addFnAttr(WaveSizeKindStr, WaveSizeStr);510 }511 // HLSL entry functions are materialized for module functions with512 // HLSLShaderAttr attribute. SetLLVMFunctionAttributesForDefinition called513 // later in the compiler-flow for such module functions is not aware of and514 // hence not able to set attributes of the newly materialized entry functions.515 // So, set attributes of entry function here, as appropriate.516 if (CGM.getCodeGenOpts().OptimizationLevel == 0)517 Fn->addFnAttr(llvm::Attribute::OptimizeNone);518 Fn->addFnAttr(llvm::Attribute::NoInline);519 520 if (CGM.getLangOpts().HLSLSpvEnableMaximalReconvergence) {521 Fn->addFnAttr("enable-maximal-reconvergence", "true");522 }523}524 525static Value *buildVectorInput(IRBuilder<> &B, Function *F, llvm::Type *Ty) {526 if (const auto *VT = dyn_cast<FixedVectorType>(Ty)) {527 Value *Result = PoisonValue::get(Ty);528 for (unsigned I = 0; I < VT->getNumElements(); ++I) {529 Value *Elt = B.CreateCall(F, {B.getInt32(I)});530 Result = B.CreateInsertElement(Result, Elt, I);531 }532 return Result;533 }534 return B.CreateCall(F, {B.getInt32(0)});535}536 537static void addSPIRVBuiltinDecoration(llvm::GlobalVariable *GV,538 unsigned BuiltIn) {539 LLVMContext &Ctx = GV->getContext();540 IRBuilder<> B(GV->getContext());541 MDNode *Operands = MDNode::get(542 Ctx,543 {ConstantAsMetadata::get(B.getInt32(/* Spirv::Decoration::BuiltIn */ 11)),544 ConstantAsMetadata::get(B.getInt32(BuiltIn))});545 MDNode *Decoration = MDNode::get(Ctx, {Operands});546 GV->addMetadata("spirv.Decorations", *Decoration);547}548 549static void addLocationDecoration(llvm::GlobalVariable *GV, unsigned Location) {550 LLVMContext &Ctx = GV->getContext();551 IRBuilder<> B(GV->getContext());552 MDNode *Operands =553 MDNode::get(Ctx, {ConstantAsMetadata::get(B.getInt32(/* Location */ 30)),554 ConstantAsMetadata::get(B.getInt32(Location))});555 MDNode *Decoration = MDNode::get(Ctx, {Operands});556 GV->addMetadata("spirv.Decorations", *Decoration);557}558 559static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M,560 llvm::Type *Ty, const Twine &Name,561 unsigned BuiltInID) {562 auto *GV = new llvm::GlobalVariable(563 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,564 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,565 llvm::GlobalVariable::GeneralDynamicTLSModel,566 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);567 addSPIRVBuiltinDecoration(GV, BuiltInID);568 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);569 return B.CreateLoad(Ty, GV);570}571 572static llvm::Value *createSPIRVLocationLoad(IRBuilder<> &B, llvm::Module &M,573 llvm::Type *Ty, unsigned Location,574 StringRef Name) {575 auto *GV = new llvm::GlobalVariable(576 M, Ty, /* isConstant= */ true, llvm::GlobalValue::ExternalLinkage,577 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,578 llvm::GlobalVariable::GeneralDynamicTLSModel,579 /* AddressSpace */ 7, /* isExternallyInitialized= */ true);580 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);581 addLocationDecoration(GV, Location);582 return B.CreateLoad(Ty, GV);583}584 585llvm::Value *586CGHLSLRuntime::emitSPIRVUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,587 HLSLAppliedSemanticAttr *Semantic,588 std::optional<unsigned> Index) {589 Twine BaseName = Twine(Semantic->getAttrName()->getName());590 Twine VariableName = BaseName.concat(Twine(Index.value_or(0)));591 592 unsigned Location = SPIRVLastAssignedInputSemanticLocation;593 594 // DXC completely ignores the semantic/index pair. Location are assigned from595 // the first semantic to the last.596 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Type);597 unsigned ElementCount = AT ? AT->getNumElements() : 1;598 SPIRVLastAssignedInputSemanticLocation += ElementCount;599 return createSPIRVLocationLoad(B, CGM.getModule(), Type, Location,600 VariableName.str());601}602 603static void createSPIRVLocationStore(IRBuilder<> &B, llvm::Module &M,604 llvm::Value *Source, unsigned Location,605 StringRef Name) {606 auto *GV = new llvm::GlobalVariable(607 M, Source->getType(), /* isConstant= */ false,608 llvm::GlobalValue::ExternalLinkage,609 /* Initializer= */ nullptr, /* Name= */ Name, /* insertBefore= */ nullptr,610 llvm::GlobalVariable::GeneralDynamicTLSModel,611 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);612 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);613 addLocationDecoration(GV, Location);614 B.CreateStore(Source, GV);615}616 617void CGHLSLRuntime::emitSPIRVUserSemanticStore(618 llvm::IRBuilder<> &B, llvm::Value *Source,619 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index) {620 Twine BaseName = Twine(Semantic->getAttrName()->getName());621 Twine VariableName = BaseName.concat(Twine(Index.value_or(0)));622 unsigned Location = SPIRVLastAssignedOutputSemanticLocation;623 624 // DXC completely ignores the semantic/index pair. Location are assigned from625 // the first semantic to the last.626 llvm::ArrayType *AT = dyn_cast<llvm::ArrayType>(Source->getType());627 unsigned ElementCount = AT ? AT->getNumElements() : 1;628 SPIRVLastAssignedOutputSemanticLocation += ElementCount;629 createSPIRVLocationStore(B, CGM.getModule(), Source, Location,630 VariableName.str());631}632 633llvm::Value *634CGHLSLRuntime::emitDXILUserSemanticLoad(llvm::IRBuilder<> &B, llvm::Type *Type,635 HLSLAppliedSemanticAttr *Semantic,636 std::optional<unsigned> Index) {637 Twine BaseName = Twine(Semantic->getAttrName()->getName());638 Twine VariableName = BaseName.concat(Twine(Index.value_or(0)));639 640 // DXIL packing rules etc shall be handled here.641 // FIXME: generate proper sigpoint, index, col, row values.642 // FIXME: also DXIL loads vectors element by element.643 SmallVector<Value *> Args{B.getInt32(4), B.getInt32(0), B.getInt32(0),644 B.getInt8(0),645 llvm::PoisonValue::get(B.getInt32Ty())};646 647 llvm::Intrinsic::ID IntrinsicID = llvm::Intrinsic::dx_load_input;648 llvm::Value *Value = B.CreateIntrinsic(/*ReturnType=*/Type, IntrinsicID, Args,649 nullptr, VariableName);650 return Value;651}652 653void CGHLSLRuntime::emitDXILUserSemanticStore(llvm::IRBuilder<> &B,654 llvm::Value *Source,655 HLSLAppliedSemanticAttr *Semantic,656 std::optional<unsigned> Index) {657 // DXIL packing rules etc shall be handled here.658 // FIXME: generate proper sigpoint, index, col, row values.659 SmallVector<Value *> Args{B.getInt32(4),660 B.getInt32(0),661 B.getInt32(0),662 B.getInt8(0),663 llvm::PoisonValue::get(B.getInt32Ty()),664 Source};665 666 llvm::Intrinsic::ID IntrinsicID = llvm::Intrinsic::dx_store_output;667 B.CreateIntrinsic(/*ReturnType=*/CGM.VoidTy, IntrinsicID, Args, nullptr);668}669 670llvm::Value *CGHLSLRuntime::emitUserSemanticLoad(671 IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,672 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index) {673 if (CGM.getTarget().getTriple().isSPIRV())674 return emitSPIRVUserSemanticLoad(B, Type, Semantic, Index);675 676 if (CGM.getTarget().getTriple().isDXIL())677 return emitDXILUserSemanticLoad(B, Type, Semantic, Index);678 679 llvm_unreachable("Unsupported target for user-semantic load.");680}681 682void CGHLSLRuntime::emitUserSemanticStore(IRBuilder<> &B, llvm::Value *Source,683 const clang::DeclaratorDecl *Decl,684 HLSLAppliedSemanticAttr *Semantic,685 std::optional<unsigned> Index) {686 if (CGM.getTarget().getTriple().isSPIRV())687 return emitSPIRVUserSemanticStore(B, Source, Semantic, Index);688 689 if (CGM.getTarget().getTriple().isDXIL())690 return emitDXILUserSemanticStore(B, Source, Semantic, Index);691 692 llvm_unreachable("Unsupported target for user-semantic load.");693}694 695llvm::Value *CGHLSLRuntime::emitSystemSemanticLoad(696 IRBuilder<> &B, llvm::Type *Type, const clang::DeclaratorDecl *Decl,697 HLSLAppliedSemanticAttr *Semantic, std::optional<unsigned> Index) {698 699 std::string SemanticName = Semantic->getAttrName()->getName().upper();700 if (SemanticName == "SV_GROUPINDEX") {701 llvm::Function *GroupIndex =702 CGM.getIntrinsic(getFlattenedThreadIdInGroupIntrinsic());703 return B.CreateCall(FunctionCallee(GroupIndex));704 }705 706 if (SemanticName == "SV_DISPATCHTHREADID") {707 llvm::Intrinsic::ID IntrinID = getThreadIdIntrinsic();708 llvm::Function *ThreadIDIntrinsic =709 llvm::Intrinsic::isOverloaded(IntrinID)710 ? CGM.getIntrinsic(IntrinID, {CGM.Int32Ty})711 : CGM.getIntrinsic(IntrinID);712 return buildVectorInput(B, ThreadIDIntrinsic, Type);713 }714 715 if (SemanticName == "SV_GROUPTHREADID") {716 llvm::Intrinsic::ID IntrinID = getGroupThreadIdIntrinsic();717 llvm::Function *GroupThreadIDIntrinsic =718 llvm::Intrinsic::isOverloaded(IntrinID)719 ? CGM.getIntrinsic(IntrinID, {CGM.Int32Ty})720 : CGM.getIntrinsic(IntrinID);721 return buildVectorInput(B, GroupThreadIDIntrinsic, Type);722 }723 724 if (SemanticName == "SV_GROUPID") {725 llvm::Intrinsic::ID IntrinID = getGroupIdIntrinsic();726 llvm::Function *GroupIDIntrinsic =727 llvm::Intrinsic::isOverloaded(IntrinID)728 ? CGM.getIntrinsic(IntrinID, {CGM.Int32Ty})729 : CGM.getIntrinsic(IntrinID);730 return buildVectorInput(B, GroupIDIntrinsic, Type);731 }732 733 if (SemanticName == "SV_POSITION") {734 if (CGM.getTriple().getEnvironment() == Triple::EnvironmentType::Pixel) {735 if (CGM.getTarget().getTriple().isSPIRV())736 return createSPIRVBuiltinLoad(B, CGM.getModule(), Type,737 Semantic->getAttrName()->getName(),738 /* BuiltIn::FragCoord */ 15);739 if (CGM.getTarget().getTriple().isDXIL())740 return emitDXILUserSemanticLoad(B, Type, Semantic, Index);741 }742 743 if (CGM.getTriple().getEnvironment() == Triple::EnvironmentType::Vertex) {744 return emitUserSemanticLoad(B, Type, Decl, Semantic, Index);745 }746 }747 748 llvm_unreachable(749 "Load hasn't been implemented yet for this system semantic. FIXME");750}751 752static void createSPIRVBuiltinStore(IRBuilder<> &B, llvm::Module &M,753 llvm::Value *Source, const Twine &Name,754 unsigned BuiltInID) {755 auto *GV = new llvm::GlobalVariable(756 M, Source->getType(), /* isConstant= */ false,757 llvm::GlobalValue::ExternalLinkage,758 /* Initializer= */ nullptr, Name, /* insertBefore= */ nullptr,759 llvm::GlobalVariable::GeneralDynamicTLSModel,760 /* AddressSpace */ 8, /* isExternallyInitialized= */ false);761 addSPIRVBuiltinDecoration(GV, BuiltInID);762 GV->setVisibility(llvm::GlobalValue::HiddenVisibility);763 B.CreateStore(Source, GV);764}765 766void CGHLSLRuntime::emitSystemSemanticStore(IRBuilder<> &B, llvm::Value *Source,767 const clang::DeclaratorDecl *Decl,768 HLSLAppliedSemanticAttr *Semantic,769 std::optional<unsigned> Index) {770 771 std::string SemanticName = Semantic->getAttrName()->getName().upper();772 if (SemanticName == "SV_POSITION") {773 if (CGM.getTarget().getTriple().isDXIL()) {774 emitDXILUserSemanticStore(B, Source, Semantic, Index);775 return;776 }777 778 if (CGM.getTarget().getTriple().isSPIRV()) {779 createSPIRVBuiltinStore(B, CGM.getModule(), Source,780 Semantic->getAttrName()->getName(),781 /* BuiltIn::Position */ 0);782 return;783 }784 }785 786 llvm_unreachable(787 "Store hasn't been implemented yet for this system semantic. FIXME");788}789 790llvm::Value *CGHLSLRuntime::handleScalarSemanticLoad(791 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,792 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic) {793 794 std::optional<unsigned> Index = Semantic->getSemanticIndex();795 if (Semantic->getAttrName()->getName().starts_with_insensitive("SV_"))796 return emitSystemSemanticLoad(B, Type, Decl, Semantic, Index);797 return emitUserSemanticLoad(B, Type, Decl, Semantic, Index);798}799 800void CGHLSLRuntime::handleScalarSemanticStore(801 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,802 const clang::DeclaratorDecl *Decl, HLSLAppliedSemanticAttr *Semantic) {803 std::optional<unsigned> Index = Semantic->getSemanticIndex();804 if (Semantic->getAttrName()->getName().starts_with_insensitive("SV_"))805 emitSystemSemanticStore(B, Source, Decl, Semantic, Index);806 else807 emitUserSemanticStore(B, Source, Decl, Semantic, Index);808}809 810std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>811CGHLSLRuntime::handleStructSemanticLoad(812 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,813 const clang::DeclaratorDecl *Decl,814 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,815 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {816 const llvm::StructType *ST = cast<StructType>(Type);817 const clang::RecordDecl *RD = Decl->getType()->getAsRecordDecl();818 819 assert(RD->getNumFields() == ST->getNumElements());820 821 llvm::Value *Aggregate = llvm::PoisonValue::get(Type);822 auto FieldDecl = RD->field_begin();823 for (unsigned I = 0; I < ST->getNumElements(); ++I) {824 auto [ChildValue, NextAttr] = handleSemanticLoad(825 B, FD, ST->getElementType(I), *FieldDecl, AttrBegin, AttrEnd);826 AttrBegin = NextAttr;827 assert(ChildValue);828 Aggregate = B.CreateInsertValue(Aggregate, ChildValue, I);829 ++FieldDecl;830 }831 832 return std::make_pair(Aggregate, AttrBegin);833}834 835specific_attr_iterator<HLSLAppliedSemanticAttr>836CGHLSLRuntime::handleStructSemanticStore(837 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,838 const clang::DeclaratorDecl *Decl,839 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,840 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {841 842 const llvm::StructType *ST = cast<StructType>(Source->getType());843 844 const clang::RecordDecl *RD = nullptr;845 if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(Decl))846 RD = FD->getDeclaredReturnType()->getAsRecordDecl();847 else848 RD = Decl->getType()->getAsRecordDecl();849 assert(RD);850 851 assert(RD->getNumFields() == ST->getNumElements());852 853 auto FieldDecl = RD->field_begin();854 for (unsigned I = 0; I < ST->getNumElements(); ++I) {855 llvm::Value *Extract = B.CreateExtractValue(Source, I);856 AttrBegin =857 handleSemanticStore(B, FD, Extract, *FieldDecl, AttrBegin, AttrEnd);858 }859 860 return AttrBegin;861}862 863std::pair<llvm::Value *, specific_attr_iterator<HLSLAppliedSemanticAttr>>864CGHLSLRuntime::handleSemanticLoad(865 IRBuilder<> &B, const FunctionDecl *FD, llvm::Type *Type,866 const clang::DeclaratorDecl *Decl,867 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,868 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {869 assert(AttrBegin != AttrEnd);870 if (Type->isStructTy())871 return handleStructSemanticLoad(B, FD, Type, Decl, AttrBegin, AttrEnd);872 873 HLSLAppliedSemanticAttr *Attr = *AttrBegin;874 ++AttrBegin;875 return std::make_pair(handleScalarSemanticLoad(B, FD, Type, Decl, Attr),876 AttrBegin);877}878 879specific_attr_iterator<HLSLAppliedSemanticAttr>880CGHLSLRuntime::handleSemanticStore(881 IRBuilder<> &B, const FunctionDecl *FD, llvm::Value *Source,882 const clang::DeclaratorDecl *Decl,883 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrBegin,884 specific_attr_iterator<HLSLAppliedSemanticAttr> AttrEnd) {885 assert(AttrBegin != AttrEnd);886 if (Source->getType()->isStructTy())887 return handleStructSemanticStore(B, FD, Source, Decl, AttrBegin, AttrEnd);888 889 HLSLAppliedSemanticAttr *Attr = *AttrBegin;890 ++AttrBegin;891 handleScalarSemanticStore(B, FD, Source, Decl, Attr);892 return AttrBegin;893}894 895void CGHLSLRuntime::emitEntryFunction(const FunctionDecl *FD,896 llvm::Function *Fn) {897 llvm::Module &M = CGM.getModule();898 llvm::LLVMContext &Ctx = M.getContext();899 auto *EntryTy = llvm::FunctionType::get(llvm::Type::getVoidTy(Ctx), false);900 Function *EntryFn =901 Function::Create(EntryTy, Function::ExternalLinkage, FD->getName(), &M);902 903 // Copy function attributes over, we have no argument or return attributes904 // that can be valid on the real entry.905 AttributeList NewAttrs = AttributeList::get(Ctx, AttributeList::FunctionIndex,906 Fn->getAttributes().getFnAttrs());907 EntryFn->setAttributes(NewAttrs);908 setHLSLEntryAttributes(FD, EntryFn);909 910 // Set the called function as internal linkage.911 Fn->setLinkage(GlobalValue::InternalLinkage);912 913 BasicBlock *BB = BasicBlock::Create(Ctx, "entry", EntryFn);914 IRBuilder<> B(BB);915 llvm::SmallVector<Value *> Args;916 917 SmallVector<OperandBundleDef, 1> OB;918 if (CGM.shouldEmitConvergenceTokens()) {919 assert(EntryFn->isConvergent());920 llvm::Value *I =921 B.CreateIntrinsic(llvm::Intrinsic::experimental_convergence_entry, {});922 llvm::Value *bundleArgs[] = {I};923 OB.emplace_back("convergencectrl", bundleArgs);924 }925 926 llvm::DenseMap<const DeclaratorDecl *, llvm::Value *> OutputSemantic;927 928 unsigned SRetOffset = 0;929 for (const auto &Param : Fn->args()) {930 if (Param.hasStructRetAttr()) {931 SRetOffset = 1;932 llvm::Type *VarType = Param.getParamStructRetType();933 llvm::Value *Var = B.CreateAlloca(VarType);934 OutputSemantic.try_emplace(FD, Var);935 Args.push_back(Var);936 continue;937 }938 939 const ParmVarDecl *PD = FD->getParamDecl(Param.getArgNo() - SRetOffset);940 llvm::Value *SemanticValue = nullptr;941 // FIXME: support inout/out parameters for semantics.942 if ([[maybe_unused]] HLSLParamModifierAttr *MA =943 PD->getAttr<HLSLParamModifierAttr>()) {944 llvm_unreachable("Not handled yet");945 } else {946 llvm::Type *ParamType =947 Param.hasByValAttr() ? Param.getParamByValType() : Param.getType();948 auto AttrBegin = PD->specific_attr_begin<HLSLAppliedSemanticAttr>();949 auto AttrEnd = PD->specific_attr_end<HLSLAppliedSemanticAttr>();950 auto Result =951 handleSemanticLoad(B, FD, ParamType, PD, AttrBegin, AttrEnd);952 SemanticValue = Result.first;953 if (!SemanticValue)954 return;955 if (Param.hasByValAttr()) {956 llvm::Value *Var = B.CreateAlloca(Param.getParamByValType());957 B.CreateStore(SemanticValue, Var);958 SemanticValue = Var;959 }960 }961 962 assert(SemanticValue);963 Args.push_back(SemanticValue);964 }965 966 CallInst *CI = B.CreateCall(FunctionCallee(Fn), Args, OB);967 CI->setCallingConv(Fn->getCallingConv());968 969 if (Fn->getReturnType() != CGM.VoidTy)970 OutputSemantic.try_emplace(FD, CI);971 972 for (auto &[Decl, Source] : OutputSemantic) {973 AllocaInst *AI = dyn_cast<AllocaInst>(Source);974 llvm::Value *SourceValue =975 AI ? B.CreateLoad(AI->getAllocatedType(), Source) : Source;976 977 auto AttrBegin = Decl->specific_attr_begin<HLSLAppliedSemanticAttr>();978 auto AttrEnd = Decl->specific_attr_end<HLSLAppliedSemanticAttr>();979 handleSemanticStore(B, FD, SourceValue, Decl, AttrBegin, AttrEnd);980 }981 982 B.CreateRetVoid();983 984 // Add and identify root signature to function, if applicable985 for (const Attr *Attr : FD->getAttrs()) {986 if (const auto *RSAttr = dyn_cast<RootSignatureAttr>(Attr)) {987 auto *RSDecl = RSAttr->getSignatureDecl();988 addRootSignatureMD(RSDecl->getVersion(), RSDecl->getRootElements(),989 EntryFn, M);990 }991 }992}993 994static void gatherFunctions(SmallVectorImpl<Function *> &Fns, llvm::Module &M,995 bool CtorOrDtor) {996 const auto *GV =997 M.getNamedGlobal(CtorOrDtor ? "llvm.global_ctors" : "llvm.global_dtors");998 if (!GV)999 return;1000 const auto *CA = dyn_cast<ConstantArray>(GV->getInitializer());1001 if (!CA)1002 return;1003 // The global_ctor array elements are a struct [Priority, Fn *, COMDat].1004 // HLSL neither supports priorities or COMDat values, so we will check those1005 // in an assert but not handle them.1006 1007 for (const auto &Ctor : CA->operands()) {1008 if (isa<ConstantAggregateZero>(Ctor))1009 continue;1010 ConstantStruct *CS = cast<ConstantStruct>(Ctor);1011 1012 assert(cast<ConstantInt>(CS->getOperand(0))->getValue() == 65535 &&1013 "HLSL doesn't support setting priority for global ctors.");1014 assert(isa<ConstantPointerNull>(CS->getOperand(2)) &&1015 "HLSL doesn't support COMDat for global ctors.");1016 Fns.push_back(cast<Function>(CS->getOperand(1)));1017 }1018}1019 1020void CGHLSLRuntime::generateGlobalCtorDtorCalls() {1021 llvm::Module &M = CGM.getModule();1022 SmallVector<Function *> CtorFns;1023 SmallVector<Function *> DtorFns;1024 gatherFunctions(CtorFns, M, true);1025 gatherFunctions(DtorFns, M, false);1026 1027 // Insert a call to the global constructor at the beginning of the entry block1028 // to externally exported functions. This is a bit of a hack, but HLSL allows1029 // global constructors, but doesn't support driver initialization of globals.1030 for (auto &F : M.functions()) {1031 if (!F.hasFnAttribute("hlsl.shader"))1032 continue;1033 auto *Token = getConvergenceToken(F.getEntryBlock());1034 Instruction *IP = &*F.getEntryBlock().begin();1035 SmallVector<OperandBundleDef, 1> OB;1036 if (Token) {1037 llvm::Value *bundleArgs[] = {Token};1038 OB.emplace_back("convergencectrl", bundleArgs);1039 IP = Token->getNextNode();1040 }1041 IRBuilder<> B(IP);1042 for (auto *Fn : CtorFns) {1043 auto CI = B.CreateCall(FunctionCallee(Fn), {}, OB);1044 CI->setCallingConv(Fn->getCallingConv());1045 }1046 1047 // Insert global dtors before the terminator of the last instruction1048 B.SetInsertPoint(F.back().getTerminator());1049 for (auto *Fn : DtorFns) {1050 auto CI = B.CreateCall(FunctionCallee(Fn), {}, OB);1051 CI->setCallingConv(Fn->getCallingConv());1052 }1053 }1054 1055 // No need to keep global ctors/dtors for non-lib profile after call to1056 // ctors/dtors added for entry.1057 Triple T(M.getTargetTriple());1058 if (T.getEnvironment() != Triple::EnvironmentType::Library) {1059 if (auto *GV = M.getNamedGlobal("llvm.global_ctors"))1060 GV->eraseFromParent();1061 if (auto *GV = M.getNamedGlobal("llvm.global_dtors"))1062 GV->eraseFromParent();1063 }1064}1065 1066static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,1067 Intrinsic::ID IntrID,1068 ArrayRef<llvm::Value *> Args) {1069 1070 LLVMContext &Ctx = CGM.getLLVMContext();1071 llvm::Function *InitResFunc = llvm::Function::Create(1072 llvm::FunctionType::get(CGM.VoidTy, false),1073 llvm::GlobalValue::InternalLinkage,1074 ("_init_buffer_" + GV->getName()).str(), CGM.getModule());1075 InitResFunc->addFnAttr(llvm::Attribute::AlwaysInline);1076 1077 llvm::BasicBlock *EntryBB =1078 llvm::BasicBlock::Create(Ctx, "entry", InitResFunc);1079 CGBuilderTy Builder(CGM, Ctx);1080 const DataLayout &DL = CGM.getModule().getDataLayout();1081 Builder.SetInsertPoint(EntryBB);1082 1083 // Make sure the global variable is buffer resource handle1084 llvm::Type *HandleTy = GV->getValueType();1085 assert(HandleTy->isTargetExtTy() && "unexpected type of the buffer global");1086 1087 llvm::Value *CreateHandle = Builder.CreateIntrinsic(1088 /*ReturnType=*/HandleTy, IntrID, Args, nullptr,1089 Twine(GV->getName()).concat("_h"));1090 1091 llvm::Value *HandleRef = Builder.CreateStructGEP(GV->getValueType(), GV, 0);1092 Builder.CreateAlignedStore(CreateHandle, HandleRef,1093 HandleRef->getPointerAlignment(DL));1094 Builder.CreateRetVoid();1095 1096 CGM.AddCXXGlobalInit(InitResFunc);1097}1098 1099void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,1100 llvm::GlobalVariable *GV) {1101 ResourceBindingAttrs Binding(BufDecl);1102 assert(Binding.hasBinding() &&1103 "cbuffer/tbuffer should always have resource binding attribute");1104 1105 auto *Index = llvm::ConstantInt::get(CGM.IntTy, 0);1106 auto *RangeSize = llvm::ConstantInt::get(CGM.IntTy, 1);1107 auto *Space = llvm::ConstantInt::get(CGM.IntTy, Binding.getSpace());1108 Value *Name = buildNameForResource(BufDecl->getName(), CGM);1109 1110 // buffer with explicit binding1111 if (Binding.isExplicit()) {1112 llvm::Intrinsic::ID IntrinsicID =1113 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();1114 auto *RegSlot = llvm::ConstantInt::get(CGM.IntTy, Binding.getSlot());1115 SmallVector<Value *> Args{Space, RegSlot, RangeSize, Index, Name};1116 initializeBuffer(CGM, GV, IntrinsicID, Args);1117 } else {1118 // buffer with implicit binding1119 llvm::Intrinsic::ID IntrinsicID =1120 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();1121 auto *OrderID =1122 llvm::ConstantInt::get(CGM.IntTy, Binding.getImplicitOrderID());1123 SmallVector<Value *> Args{OrderID, Space, RangeSize, Index, Name};1124 initializeBuffer(CGM, GV, IntrinsicID, Args);1125 }1126}1127 1128void CGHLSLRuntime::handleGlobalVarDefinition(const VarDecl *VD,1129 llvm::GlobalVariable *GV) {1130 if (auto Attr = VD->getAttr<HLSLVkExtBuiltinInputAttr>())1131 addSPIRVBuiltinDecoration(GV, Attr->getBuiltIn());1132}1133 1134llvm::Instruction *CGHLSLRuntime::getConvergenceToken(BasicBlock &BB) {1135 if (!CGM.shouldEmitConvergenceTokens())1136 return nullptr;1137 1138 auto E = BB.end();1139 for (auto I = BB.begin(); I != E; ++I) {1140 auto *II = dyn_cast<llvm::IntrinsicInst>(&*I);1141 if (II && llvm::isConvergenceControlIntrinsic(II->getIntrinsicID())) {1142 return II;1143 }1144 }1145 llvm_unreachable("Convergence token should have been emitted.");1146 return nullptr;1147}1148 1149class OpaqueValueVisitor : public RecursiveASTVisitor<OpaqueValueVisitor> {1150public:1151 llvm::SmallVector<OpaqueValueExpr *, 8> OVEs;1152 llvm::SmallPtrSet<OpaqueValueExpr *, 8> Visited;1153 OpaqueValueVisitor() {}1154 1155 bool VisitHLSLOutArgExpr(HLSLOutArgExpr *) {1156 // These need to be bound in CodeGenFunction::EmitHLSLOutArgLValues1157 // or CodeGenFunction::EmitHLSLOutArgExpr. If they are part of this1158 // traversal, the temporary containing the copy out will not have1159 // been created yet.1160 return false;1161 }1162 1163 bool VisitOpaqueValueExpr(OpaqueValueExpr *E) {1164 // Traverse the source expression first.1165 if (E->getSourceExpr())1166 TraverseStmt(E->getSourceExpr());1167 1168 // Then add this OVE if we haven't seen it before.1169 if (Visited.insert(E).second)1170 OVEs.push_back(E);1171 1172 return true;1173 }1174};1175 1176void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,1177 InitListExpr *E) {1178 1179 typedef CodeGenFunction::OpaqueValueMappingData OpaqueValueMappingData;1180 OpaqueValueVisitor Visitor;1181 Visitor.TraverseStmt(E);1182 for (auto *OVE : Visitor.OVEs) {1183 if (CGF.isOpaqueValueEmitted(OVE))1184 continue;1185 if (OpaqueValueMappingData::shouldBindAsLValue(OVE)) {1186 LValue LV = CGF.EmitLValue(OVE->getSourceExpr());1187 OpaqueValueMappingData::bind(CGF, OVE, LV);1188 } else {1189 RValue RV = CGF.EmitAnyExpr(OVE->getSourceExpr());1190 OpaqueValueMappingData::bind(CGF, OVE, RV);1191 }1192 }1193}1194 1195std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(1196 const ArraySubscriptExpr *ArraySubsExpr, CodeGenFunction &CGF) {1197 assert((ArraySubsExpr->getType()->isHLSLResourceRecord() ||1198 ArraySubsExpr->getType()->isHLSLResourceRecordArray()) &&1199 "expected resource array subscript expression");1200 1201 // Let clang codegen handle local resource array subscripts,1202 // or when the subscript references on opaque expression (as part of1203 // ArrayInitLoopExpr AST node).1204 const VarDecl *ArrayDecl =1205 dyn_cast_or_null<VarDecl>(getArrayDecl(ArraySubsExpr));1206 if (!ArrayDecl || !ArrayDecl->hasGlobalStorage())1207 return std::nullopt;1208 1209 // get the resource array type1210 ASTContext &AST = ArrayDecl->getASTContext();1211 const Type *ResArrayTy = ArrayDecl->getType().getTypePtr();1212 assert(ResArrayTy->isHLSLResourceRecordArray() &&1213 "expected array of resource classes");1214 1215 // Iterate through all nested array subscript expressions to calculate1216 // the index in the flattened resource array (if this is a multi-1217 // dimensional array). The index is calculated as a sum of all indices1218 // multiplied by the total size of the array at that level.1219 Value *Index = nullptr;1220 const ArraySubscriptExpr *ASE = ArraySubsExpr;1221 while (ASE != nullptr) {1222 Value *SubIndex = CGF.EmitScalarExpr(ASE->getIdx());1223 if (const auto *ArrayTy =1224 dyn_cast<ConstantArrayType>(ASE->getType().getTypePtr())) {1225 Value *Multiplier = llvm::ConstantInt::get(1226 CGM.IntTy, AST.getConstantArrayElementCount(ArrayTy));1227 SubIndex = CGF.Builder.CreateMul(SubIndex, Multiplier);1228 }1229 Index = Index ? CGF.Builder.CreateAdd(Index, SubIndex) : SubIndex;1230 ASE = dyn_cast<ArraySubscriptExpr>(ASE->getBase()->IgnoreParenImpCasts());1231 }1232 1233 // Find binding info for the resource array. For implicit binding1234 // an HLSLResourceBindingAttr should have been added by SemaHLSL.1235 ResourceBindingAttrs Binding(ArrayDecl);1236 assert((Binding.hasBinding()) &&1237 "resource array must have a binding attribute");1238 1239 // Find the individual resource type.1240 QualType ResultTy = ArraySubsExpr->getType();1241 QualType ResourceTy =1242 ResultTy->isArrayType() ? AST.getBaseElementType(ResultTy) : ResultTy;1243 1244 // Create a temporary variable for the result, which is either going1245 // to be a single resource instance or a local array of resources (we need to1246 // return an LValue).1247 RawAddress TmpVar = CGF.CreateMemTemp(ResultTy);1248 if (CGF.EmitLifetimeStart(TmpVar.getPointer()))1249 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(1250 NormalEHLifetimeMarker, TmpVar);1251 1252 AggValueSlot ValueSlot = AggValueSlot::forAddr(1253 TmpVar, Qualifiers(), AggValueSlot::IsDestructed_t(true),1254 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsAliased_t(false),1255 AggValueSlot::DoesNotOverlap);1256 1257 // Calculate total array size (= range size).1258 llvm::Value *Range =1259 llvm::ConstantInt::get(CGM.IntTy, getTotalArraySize(AST, ResArrayTy));1260 1261 // If the result of the subscript operation is a single resource, call the1262 // constructor.1263 if (ResultTy == ResourceTy) {1264 CallArgList Args;1265 CXXMethodDecl *CreateMethod = lookupResourceInitMethodAndSetupArgs(1266 CGF.CGM, ResourceTy->getAsCXXRecordDecl(), Range, Index,1267 ArrayDecl->getName(), Binding, Args);1268 1269 if (!CreateMethod)1270 // This can happen if someone creates an array of structs that looks like1271 // an HLSL resource record array but it does not have the required static1272 // create method. No binding will be generated for it.1273 return std::nullopt;1274 1275 callResourceInitMethod(CGF, CreateMethod, Args, ValueSlot.getAddress());1276 1277 } else {1278 // The result of the subscript operation is a local resource array which1279 // needs to be initialized.1280 const ConstantArrayType *ArrayTy =1281 cast<ConstantArrayType>(ResultTy.getTypePtr());1282 std::optional<llvm::Value *> EndIndex = initializeLocalResourceArray(1283 CGF, ResourceTy->getAsCXXRecordDecl(), ArrayTy, ValueSlot, Range, Index,1284 ArrayDecl->getName(), Binding, {llvm::ConstantInt::get(CGM.IntTy, 0)},1285 ArraySubsExpr->getExprLoc());1286 if (!EndIndex)1287 return std::nullopt;1288 }1289 return CGF.MakeAddrLValue(TmpVar, ResultTy, AlignmentSource::Decl);1290}1291 1292std::optional<LValue> CGHLSLRuntime::emitBufferArraySubscriptExpr(1293 const ArraySubscriptExpr *E, CodeGenFunction &CGF,1294 llvm::function_ref<llvm::Value *(bool Promote)> EmitIdxAfterBase) {1295 // Find the element type to index by first padding the element type per HLSL1296 // buffer rules, and then padding out to a 16-byte register boundary if1297 // necessary.1298 llvm::Type *LayoutTy =1299 HLSLBufferLayoutBuilder(CGF.CGM).layOutType(E->getType());1300 uint64_t LayoutSizeInBits =1301 CGM.getDataLayout().getTypeSizeInBits(LayoutTy).getFixedValue();1302 CharUnits ElementSize = CharUnits::fromQuantity(LayoutSizeInBits / 8);1303 CharUnits RowAlignedSize = ElementSize.alignTo(CharUnits::fromQuantity(16));1304 if (RowAlignedSize > ElementSize) {1305 llvm::Type *Padding = CGM.getTargetCodeGenInfo().getHLSLPadding(1306 CGM, RowAlignedSize - ElementSize);1307 assert(Padding && "No padding type for target?");1308 LayoutTy = llvm::StructType::get(CGF.getLLVMContext(), {LayoutTy, Padding},1309 /*isPacked=*/true);1310 }1311 1312 // If the layout type doesn't introduce any padding, we don't need to do1313 // anything special.1314 llvm::Type *OrigTy = CGF.CGM.getTypes().ConvertTypeForMem(E->getType());1315 if (LayoutTy == OrigTy)1316 return std::nullopt;1317 1318 LValueBaseInfo EltBaseInfo;1319 TBAAAccessInfo EltTBAAInfo;1320 Address Addr =1321 CGF.EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);1322 llvm::Value *Idx = EmitIdxAfterBase(/*Promote*/ true);1323 1324 // Index into the object as-if we have an array of the padded element type,1325 // and then dereference the element itself to avoid reading padding that may1326 // be past the end of the in-memory object.1327 SmallVector<llvm::Value *, 2> Indices;1328 Indices.push_back(Idx);1329 Indices.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 0));1330 1331 llvm::Value *GEP = CGF.Builder.CreateGEP(LayoutTy, Addr.emitRawPointer(CGF),1332 Indices, "cbufferidx");1333 Addr = Address(GEP, Addr.getElementType(), RowAlignedSize, KnownNonNull);1334 1335 return CGF.MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);1336}1337 1338namespace {1339/// Utility for emitting copies following the HLSL buffer layout rules (ie,1340/// copying out of a cbuffer).1341class HLSLBufferCopyEmitter {1342 CodeGenFunction &CGF;1343 Address DestPtr;1344 Address SrcPtr;1345 llvm::Type *LayoutTy = nullptr;1346 1347 SmallVector<llvm::Value *> CurStoreIndices;1348 SmallVector<llvm::Value *> CurLoadIndices;1349 1350 void emitCopyAtIndices(llvm::Type *FieldTy, llvm::ConstantInt *StoreIndex,1351 llvm::ConstantInt *LoadIndex) {1352 CurStoreIndices.push_back(StoreIndex);1353 CurLoadIndices.push_back(LoadIndex);1354 auto RestoreIndices = llvm::make_scope_exit([&]() {1355 CurStoreIndices.pop_back();1356 CurLoadIndices.pop_back();1357 });1358 1359 // First, see if this is some kind of aggregate and recurse.1360 if (processArray(FieldTy))1361 return;1362 if (processBufferLayoutArray(FieldTy))1363 return;1364 if (processStruct(FieldTy))1365 return;1366 1367 // When we have a scalar or vector element we can emit the copy.1368 CharUnits Align = CharUnits::fromQuantity(1369 CGF.CGM.getDataLayout().getABITypeAlign(FieldTy));1370 Address SrcGEP = RawAddress(1371 CGF.Builder.CreateInBoundsGEP(LayoutTy, SrcPtr.getBasePointer(),1372 CurLoadIndices, "cbuf.src"),1373 FieldTy, Align, SrcPtr.isKnownNonNull());1374 Address DestGEP = CGF.Builder.CreateInBoundsGEP(1375 DestPtr, CurStoreIndices, FieldTy, Align, "cbuf.dest");1376 llvm::Value *Load = CGF.Builder.CreateLoad(SrcGEP, "cbuf.load");1377 CGF.Builder.CreateStore(Load, DestGEP);1378 }1379 1380 bool processArray(llvm::Type *FieldTy) {1381 auto *AT = dyn_cast<llvm::ArrayType>(FieldTy);1382 if (!AT)1383 return false;1384 1385 // If we have an llvm::ArrayType this is just a regular array with no top1386 // level padding, so all we need to do is copy each member.1387 for (unsigned I = 0, E = AT->getNumElements(); I < E; ++I)1388 emitCopyAtIndices(AT->getElementType(),1389 llvm::ConstantInt::get(CGF.SizeTy, I),1390 llvm::ConstantInt::get(CGF.SizeTy, I));1391 return true;1392 }1393 1394 bool processBufferLayoutArray(llvm::Type *FieldTy) {1395 // A buffer layout array is a struct with two elements: the padded array,1396 // and the last element. That is, is should look something like this:1397 //1398 // { [%n x { %type, %padding }], %type }1399 //1400 auto *ST = dyn_cast<llvm::StructType>(FieldTy);1401 if (!ST || ST->getNumElements() != 2)1402 return false;1403 1404 auto *PaddedEltsTy = dyn_cast<llvm::ArrayType>(ST->getElementType(0));1405 if (!PaddedEltsTy)1406 return false;1407 1408 auto *PaddedTy = dyn_cast<llvm::StructType>(PaddedEltsTy->getElementType());1409 if (!PaddedTy || PaddedTy->getNumElements() != 2)1410 return false;1411 1412 if (!CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(1413 PaddedTy->getElementType(1)))1414 return false;1415 1416 llvm::Type *ElementTy = ST->getElementType(1);1417 if (PaddedTy->getElementType(0) != ElementTy)1418 return false;1419 1420 // All but the last of the logical array elements are in the padded array.1421 unsigned NumElts = PaddedEltsTy->getNumElements() + 1;1422 1423 // Add an extra indirection to the load for the struct and walk the1424 // array prefix.1425 CurLoadIndices.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 0));1426 for (unsigned I = 0; I < NumElts - 1; ++I) {1427 // We need to copy the element itself, without the padding.1428 CurLoadIndices.push_back(llvm::ConstantInt::get(CGF.SizeTy, I));1429 emitCopyAtIndices(ElementTy, llvm::ConstantInt::get(CGF.SizeTy, I),1430 llvm::ConstantInt::get(CGF.Int32Ty, 0));1431 CurLoadIndices.pop_back();1432 }1433 CurLoadIndices.pop_back();1434 1435 // Now copy the last element.1436 emitCopyAtIndices(ElementTy,1437 llvm::ConstantInt::get(CGF.SizeTy, NumElts - 1),1438 llvm::ConstantInt::get(CGF.Int32Ty, 1));1439 1440 return true;1441 }1442 1443 bool processStruct(llvm::Type *FieldTy) {1444 auto *ST = dyn_cast<llvm::StructType>(FieldTy);1445 if (!ST)1446 return false;1447 1448 // Copy the struct field by field, but skip any explicit padding.1449 unsigned Skipped = 0;1450 for (unsigned I = 0, E = ST->getNumElements(); I < E; ++I) {1451 llvm::Type *ElementTy = ST->getElementType(I);1452 if (CGF.CGM.getTargetCodeGenInfo().isHLSLPadding(ElementTy))1453 ++Skipped;1454 else1455 emitCopyAtIndices(ElementTy, llvm::ConstantInt::get(CGF.Int32Ty, I),1456 llvm::ConstantInt::get(CGF.Int32Ty, I + Skipped));1457 }1458 return true;1459 }1460 1461public:1462 HLSLBufferCopyEmitter(CodeGenFunction &CGF, Address DestPtr, Address SrcPtr)1463 : CGF(CGF), DestPtr(DestPtr), SrcPtr(SrcPtr) {}1464 1465 bool emitCopy(QualType CType) {1466 LayoutTy = HLSLBufferLayoutBuilder(CGF.CGM).layOutType(CType);1467 1468 // TODO: We should be able to fall back to a regular memcpy if the layout1469 // type doesn't have any padding, but that runs into issues in the backend1470 // currently.1471 //1472 // See https://github.com/llvm/wg-hlsl/issues/3511473 emitCopyAtIndices(LayoutTy, llvm::ConstantInt::get(CGF.SizeTy, 0),1474 llvm::ConstantInt::get(CGF.SizeTy, 0));1475 return true;1476 }1477};1478} // namespace1479 1480bool CGHLSLRuntime::emitBufferCopy(CodeGenFunction &CGF, Address DestPtr,1481 Address SrcPtr, QualType CType) {1482 return HLSLBufferCopyEmitter(CGF, DestPtr, SrcPtr).emitCopy(CType);1483}1484 1485LValue CGHLSLRuntime::emitBufferMemberExpr(CodeGenFunction &CGF,1486 const MemberExpr *E) {1487 LValue Base =1488 CGF.EmitCheckedLValue(E->getBase(), CodeGenFunction::TCK_MemberAccess);1489 auto *Field = dyn_cast<FieldDecl>(E->getMemberDecl());1490 assert(Field && "Unexpected access into HLSL buffer");1491 1492 // Get the field index for the struct.1493 const RecordDecl *Rec = Field->getParent();1494 unsigned FieldIdx =1495 CGM.getTypes().getCGRecordLayout(Rec).getLLVMFieldNo(Field);1496 1497 // Work out the buffer layout type to index into.1498 QualType RecType = CGM.getContext().getCanonicalTagType(Rec);1499 assert(RecType->isStructureOrClassType() && "Invalid type in HLSL buffer");1500 // Since this is a member of an object in the buffer and not the buffer's1501 // struct/class itself, we shouldn't have any offsets on the members we need1502 // to contend with.1503 CGHLSLOffsetInfo EmptyOffsets;1504 llvm::StructType *LayoutTy = HLSLBufferLayoutBuilder(CGM).layOutStruct(1505 RecType->getAsCanonical<RecordType>(), EmptyOffsets);1506 1507 // Now index into the struct, making sure that the type we return is the1508 // buffer layout type rather than the original type in the AST.1509 QualType FieldType = Field->getType();1510 llvm::Type *FieldLLVMTy = CGM.getTypes().ConvertTypeForMem(FieldType);1511 CharUnits Align = CharUnits::fromQuantity(1512 CGF.CGM.getDataLayout().getABITypeAlign(FieldLLVMTy));1513 Address Addr(CGF.Builder.CreateStructGEP(LayoutTy, Base.getPointer(CGF),1514 FieldIdx, Field->getName()),1515 FieldLLVMTy, Align, KnownNonNull);1516 1517 LValue LV = LValue::MakeAddr(Addr, FieldType, CGM.getContext(),1518 LValueBaseInfo(AlignmentSource::Type),1519 CGM.getTBAAAccessInfo(FieldType));1520 LV.getQuals().addCVRQualifiers(Base.getVRQualifiers());1521 1522 return LV;1523}1524