1027 lines · cpp
1//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This contains code to emit HLSL Builtin calls as LLVM code.10//11//===----------------------------------------------------------------------===//12 13#include "CGBuiltin.h"14#include "CGHLSLRuntime.h"15#include "CodeGenFunction.h"16 17using namespace clang;18using namespace CodeGen;19using namespace llvm;20 21static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {22 assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&23 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&24 "asdouble operands types mismatch");25 Value *OpLowBits = CGF.EmitScalarExpr(E->getArg(0));26 Value *OpHighBits = CGF.EmitScalarExpr(E->getArg(1));27 28 llvm::Type *ResultType = CGF.DoubleTy;29 int N = 1;30 if (auto *VTy = E->getArg(0)->getType()->getAs<clang::VectorType>()) {31 N = VTy->getNumElements();32 ResultType = llvm::FixedVectorType::get(CGF.DoubleTy, N);33 }34 35 if (CGF.CGM.getTarget().getTriple().isDXIL())36 return CGF.Builder.CreateIntrinsic(37 /*ReturnType=*/ResultType, Intrinsic::dx_asdouble,38 {OpLowBits, OpHighBits}, nullptr, "hlsl.asdouble");39 40 if (!E->getArg(0)->getType()->isVectorType()) {41 OpLowBits = CGF.Builder.CreateVectorSplat(1, OpLowBits);42 OpHighBits = CGF.Builder.CreateVectorSplat(1, OpHighBits);43 }44 45 llvm::SmallVector<int> Mask;46 for (int i = 0; i < N; i++) {47 Mask.push_back(i);48 Mask.push_back(i + N);49 }50 51 Value *BitVec = CGF.Builder.CreateShuffleVector(OpLowBits, OpHighBits, Mask);52 53 return CGF.Builder.CreateBitCast(BitVec, ResultType);54}55 56static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {57 Value *Op0 = CGF->EmitScalarExpr(E->getArg(0));58 59 Constant *FZeroConst = ConstantFP::getZero(CGF->FloatTy);60 Value *CMP;61 Value *LastInstr;62 63 if (const auto *VecTy = E->getArg(0)->getType()->getAs<clang::VectorType>()) {64 FZeroConst = ConstantVector::getSplat(65 ElementCount::getFixed(VecTy->getNumElements()), FZeroConst);66 auto *FCompInst = CGF->Builder.CreateFCmpOLT(Op0, FZeroConst);67 CMP = CGF->Builder.CreateIntrinsic(68 CGF->Builder.getInt1Ty(), CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),69 {FCompInst});70 } else {71 CMP = CGF->Builder.CreateFCmpOLT(Op0, FZeroConst);72 }73 74 if (CGF->CGM.getTarget().getTriple().isDXIL()) {75 LastInstr = CGF->Builder.CreateIntrinsic(Intrinsic::dx_discard, {CMP});76 } else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {77 BasicBlock *LT0 = CGF->createBasicBlock("lt0", CGF->CurFn);78 BasicBlock *End = CGF->createBasicBlock("end", CGF->CurFn);79 80 CGF->Builder.CreateCondBr(CMP, LT0, End);81 82 CGF->Builder.SetInsertPoint(LT0);83 84 CGF->Builder.CreateIntrinsic(Intrinsic::spv_discard, {});85 86 LastInstr = CGF->Builder.CreateBr(End);87 CGF->Builder.SetInsertPoint(End);88 } else {89 llvm_unreachable("Backend Codegen not supported.");90 }91 92 return LastInstr;93}94 95static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {96 Value *Op0 = CGF->EmitScalarExpr(E->getArg(0));97 const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(E->getArg(1));98 const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(E->getArg(2));99 100 CallArgList Args;101 LValue Op1TmpLValue =102 CGF->EmitHLSLOutArgExpr(OutArg1, Args, OutArg1->getType());103 LValue Op2TmpLValue =104 CGF->EmitHLSLOutArgExpr(OutArg2, Args, OutArg2->getType());105 106 if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())107 Args.reverseWritebacks();108 109 Value *LowBits = nullptr;110 Value *HighBits = nullptr;111 112 if (CGF->CGM.getTarget().getTriple().isDXIL()) {113 llvm::Type *RetElementTy = CGF->Int32Ty;114 if (auto *Op0VecTy = E->getArg(0)->getType()->getAs<clang::VectorType>())115 RetElementTy = llvm::VectorType::get(116 CGF->Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements()));117 auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy);118 119 CallInst *CI = CGF->Builder.CreateIntrinsic(120 RetTy, Intrinsic::dx_splitdouble, {Op0}, nullptr, "hlsl.splitdouble");121 122 LowBits = CGF->Builder.CreateExtractValue(CI, 0);123 HighBits = CGF->Builder.CreateExtractValue(CI, 1);124 } else {125 // For Non DXIL targets we generate the instructions.126 127 if (!Op0->getType()->isVectorTy()) {128 FixedVectorType *DestTy = FixedVectorType::get(CGF->Int32Ty, 2);129 Value *Bitcast = CGF->Builder.CreateBitCast(Op0, DestTy);130 131 LowBits = CGF->Builder.CreateExtractElement(Bitcast, (uint64_t)0);132 HighBits = CGF->Builder.CreateExtractElement(Bitcast, 1);133 } else {134 int NumElements = 1;135 if (const auto *VecTy =136 E->getArg(0)->getType()->getAs<clang::VectorType>())137 NumElements = VecTy->getNumElements();138 139 FixedVectorType *Uint32VecTy =140 FixedVectorType::get(CGF->Int32Ty, NumElements * 2);141 Value *Uint32Vec = CGF->Builder.CreateBitCast(Op0, Uint32VecTy);142 if (NumElements == 1) {143 LowBits = CGF->Builder.CreateExtractElement(Uint32Vec, (uint64_t)0);144 HighBits = CGF->Builder.CreateExtractElement(Uint32Vec, 1);145 } else {146 SmallVector<int> EvenMask, OddMask;147 for (int I = 0, E = NumElements; I != E; ++I) {148 EvenMask.push_back(I * 2);149 OddMask.push_back(I * 2 + 1);150 }151 LowBits = CGF->Builder.CreateShuffleVector(Uint32Vec, EvenMask);152 HighBits = CGF->Builder.CreateShuffleVector(Uint32Vec, OddMask);153 }154 }155 }156 CGF->Builder.CreateStore(LowBits, Op1TmpLValue.getAddress());157 auto *LastInst =158 CGF->Builder.CreateStore(HighBits, Op2TmpLValue.getAddress());159 CGF->EmitWritebacks(Args);160 return LastInst;161}162 163static Value *handleElementwiseF16ToF32(CodeGenFunction &CGF,164 const CallExpr *E) {165 Value *Op0 = CGF.EmitScalarExpr(E->getArg(0));166 QualType Op0Ty = E->getArg(0)->getType();167 llvm::Type *ResType = CGF.FloatTy;168 uint64_t NumElements = 0;169 if (Op0->getType()->isVectorTy()) {170 NumElements =171 E->getArg(0)->getType()->castAs<clang::VectorType>()->getNumElements();172 ResType =173 llvm::VectorType::get(ResType, ElementCount::getFixed(NumElements));174 }175 if (!Op0Ty->hasUnsignedIntegerRepresentation())176 llvm_unreachable(177 "f16tof32 operand must have an unsigned int representation");178 179 if (CGF.CGM.getTriple().isDXIL())180 return CGF.Builder.CreateIntrinsic(ResType, Intrinsic::dx_legacyf16tof32,181 ArrayRef<Value *>{Op0}, nullptr,182 "hlsl.f16tof32");183 184 if (CGF.CGM.getTriple().isSPIRV()) {185 // We use the SPIRV UnpackHalf2x16 operation to avoid the need for the186 // Int16 and Float16 capabilities187 auto UnpackType =188 llvm::VectorType::get(CGF.FloatTy, ElementCount::getFixed(2));189 if (NumElements == 0) {190 // a scalar input - simply extract the first element of the unpacked191 // vector192 Value *Unpack = CGF.Builder.CreateIntrinsic(193 UnpackType, Intrinsic::spv_unpackhalf2x16, ArrayRef<Value *>{Op0});194 return CGF.Builder.CreateExtractElement(Unpack, (uint64_t)0);195 } else {196 // a vector input - build a congruent output vector by iterating through197 // the input vector calling unpackhalf2x16 for each element198 Value *Result = PoisonValue::get(ResType);199 for (uint64_t i = 0; i < NumElements; i++) {200 Value *InVal = CGF.Builder.CreateExtractElement(Op0, i);201 Value *Unpack = CGF.Builder.CreateIntrinsic(202 UnpackType, Intrinsic::spv_unpackhalf2x16,203 ArrayRef<Value *>{InVal});204 Value *Res = CGF.Builder.CreateExtractElement(Unpack, (uint64_t)0);205 Result = CGF.Builder.CreateInsertElement(Result, Res, i);206 }207 return Result;208 }209 }210 211 llvm_unreachable("Intrinsic F16ToF32 not supported by target architecture");212}213 214static Value *emitBufferStride(CodeGenFunction *CGF, const Expr *HandleExpr,215 LValue &Stride) {216 // Figure out the stride of the buffer elements from the handle type.217 auto *HandleTy =218 cast<HLSLAttributedResourceType>(HandleExpr->getType().getTypePtr());219 QualType ElementTy = HandleTy->getContainedType();220 Value *StrideValue = CGF->getTypeSize(ElementTy);221 return CGF->Builder.CreateStore(StrideValue, Stride.getAddress());222}223 224// Return dot product intrinsic that corresponds to the QT scalar type225static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {226 if (QT->isFloatingType())227 return RT.getFDotIntrinsic();228 if (QT->isSignedIntegerType())229 return RT.getSDotIntrinsic();230 assert(QT->isUnsignedIntegerType());231 return RT.getUDotIntrinsic();232}233 234static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {235 if (QT->hasSignedIntegerRepresentation()) {236 return RT.getFirstBitSHighIntrinsic();237 }238 239 assert(QT->hasUnsignedIntegerRepresentation());240 return RT.getFirstBitUHighIntrinsic();241}242 243// Return wave active sum that corresponds to the QT scalar type244static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,245 CGHLSLRuntime &RT, QualType QT) {246 switch (Arch) {247 case llvm::Triple::spirv:248 return Intrinsic::spv_wave_reduce_sum;249 case llvm::Triple::dxil: {250 if (QT->isUnsignedIntegerType())251 return Intrinsic::dx_wave_reduce_usum;252 return Intrinsic::dx_wave_reduce_sum;253 }254 default:255 llvm_unreachable("Intrinsic WaveActiveSum"256 " not supported by target architecture");257 }258}259 260// Return wave active max that corresponds to the QT scalar type261static Intrinsic::ID getWaveActiveMaxIntrinsic(llvm::Triple::ArchType Arch,262 CGHLSLRuntime &RT, QualType QT) {263 switch (Arch) {264 case llvm::Triple::spirv:265 if (QT->isUnsignedIntegerType())266 return Intrinsic::spv_wave_reduce_umax;267 return Intrinsic::spv_wave_reduce_max;268 case llvm::Triple::dxil: {269 if (QT->isUnsignedIntegerType())270 return Intrinsic::dx_wave_reduce_umax;271 return Intrinsic::dx_wave_reduce_max;272 }273 default:274 llvm_unreachable("Intrinsic WaveActiveMax"275 " not supported by target architecture");276 }277}278 279// Return wave active min that corresponds to the QT scalar type280static Intrinsic::ID getWaveActiveMinIntrinsic(llvm::Triple::ArchType Arch,281 CGHLSLRuntime &RT, QualType QT) {282 switch (Arch) {283 case llvm::Triple::spirv:284 if (QT->isUnsignedIntegerType())285 return Intrinsic::spv_wave_reduce_umin;286 return Intrinsic::spv_wave_reduce_min;287 case llvm::Triple::dxil: {288 if (QT->isUnsignedIntegerType())289 return Intrinsic::dx_wave_reduce_umin;290 return Intrinsic::dx_wave_reduce_min;291 }292 default:293 llvm_unreachable("Intrinsic WaveActiveMin"294 " not supported by target architecture");295 }296}297 298// Returns the mangled name for a builtin function that the SPIR-V backend299// will expand into a spec Constant.300static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,301 ASTContext &Context) {302 // The parameter types for our conceptual intrinsic function.303 QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};304 305 // Create a temporary FunctionDecl for the builtin fuction. It won't be306 // added to the AST.307 FunctionProtoType::ExtProtoInfo EPI;308 QualType FnType =309 Context.getFunctionType(SpecConstantType, ClangParamTypes, EPI);310 DeclarationName FuncName = &Context.Idents.get("__spirv_SpecConstant");311 FunctionDecl *FnDeclForMangling = FunctionDecl::Create(312 Context, Context.getTranslationUnitDecl(), SourceLocation(),313 SourceLocation(), FuncName, FnType, /*TSI=*/nullptr, SC_Extern);314 315 // Attach the created parameter declarations to the function declaration.316 SmallVector<ParmVarDecl *, 2> ParamDecls;317 for (QualType ParamType : ClangParamTypes) {318 ParmVarDecl *PD = ParmVarDecl::Create(319 Context, FnDeclForMangling, SourceLocation(), SourceLocation(),320 /*IdentifierInfo*/ nullptr, ParamType, /*TSI*/ nullptr, SC_None,321 /*DefaultArg*/ nullptr);322 ParamDecls.push_back(PD);323 }324 FnDeclForMangling->setParams(ParamDecls);325 326 // Get the mangled name.327 std::string Name;328 llvm::raw_string_ostream MangledNameStream(Name);329 std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());330 Mangler->mangleName(FnDeclForMangling, MangledNameStream);331 MangledNameStream.flush();332 333 return Name;334}335 336Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,337 const CallExpr *E,338 ReturnValueSlot ReturnValue) {339 if (!getLangOpts().HLSL)340 return nullptr;341 342 switch (BuiltinID) {343 case Builtin::BI__builtin_hlsl_adduint64: {344 Value *OpA = EmitScalarExpr(E->getArg(0));345 Value *OpB = EmitScalarExpr(E->getArg(1));346 QualType Arg0Ty = E->getArg(0)->getType();347 uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();348 assert(Arg0Ty == E->getArg(1)->getType() &&349 "AddUint64 operand types must match");350 assert(Arg0Ty->hasIntegerRepresentation() &&351 "AddUint64 operands must have an integer representation");352 assert((NumElements == 2 || NumElements == 4) &&353 "AddUint64 operands must have 2 or 4 elements");354 355 llvm::Value *LowA;356 llvm::Value *HighA;357 llvm::Value *LowB;358 llvm::Value *HighB;359 360 // Obtain low and high words of inputs A and B361 if (NumElements == 2) {362 LowA = Builder.CreateExtractElement(OpA, (uint64_t)0, "LowA");363 HighA = Builder.CreateExtractElement(OpA, (uint64_t)1, "HighA");364 LowB = Builder.CreateExtractElement(OpB, (uint64_t)0, "LowB");365 HighB = Builder.CreateExtractElement(OpB, (uint64_t)1, "HighB");366 } else {367 LowA = Builder.CreateShuffleVector(OpA, {0, 2}, "LowA");368 HighA = Builder.CreateShuffleVector(OpA, {1, 3}, "HighA");369 LowB = Builder.CreateShuffleVector(OpB, {0, 2}, "LowB");370 HighB = Builder.CreateShuffleVector(OpB, {1, 3}, "HighB");371 }372 373 // Use an uadd_with_overflow to compute the sum of low words and obtain a374 // carry value375 llvm::Value *Carry;376 llvm::Value *LowSum = EmitOverflowIntrinsic(377 *this, Intrinsic::uadd_with_overflow, LowA, LowB, Carry);378 llvm::Value *ZExtCarry =379 Builder.CreateZExt(Carry, HighA->getType(), "CarryZExt");380 381 // Sum the high words and the carry382 llvm::Value *HighSum = Builder.CreateAdd(HighA, HighB, "HighSum");383 llvm::Value *HighSumPlusCarry =384 Builder.CreateAdd(HighSum, ZExtCarry, "HighSumPlusCarry");385 386 if (NumElements == 4) {387 return Builder.CreateShuffleVector(LowSum, HighSumPlusCarry, {0, 2, 1, 3},388 "hlsl.AddUint64");389 }390 391 llvm::Value *Result = PoisonValue::get(OpA->getType());392 Result = Builder.CreateInsertElement(Result, LowSum, (uint64_t)0,393 "hlsl.AddUint64.upto0");394 Result = Builder.CreateInsertElement(Result, HighSumPlusCarry, (uint64_t)1,395 "hlsl.AddUint64");396 return Result;397 }398 case Builtin::BI__builtin_hlsl_resource_getpointer: {399 Value *HandleOp = EmitScalarExpr(E->getArg(0));400 Value *IndexOp = EmitScalarExpr(E->getArg(1));401 402 llvm::Type *RetTy = ConvertType(E->getType());403 return Builder.CreateIntrinsic(404 RetTy, CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),405 ArrayRef<Value *>{HandleOp, IndexOp});406 }407 case Builtin::BI__builtin_hlsl_resource_load_with_status: {408 Value *HandleOp = EmitScalarExpr(E->getArg(0));409 Value *IndexOp = EmitScalarExpr(E->getArg(1));410 411 // Get the *address* of the status argument to write to it by reference412 LValue StatusLVal = EmitLValue(E->getArg(2));413 Address StatusAddr = StatusLVal.getAddress();414 415 QualType HandleTy = E->getArg(0)->getType();416 const HLSLAttributedResourceType *RT =417 HandleTy->getAs<HLSLAttributedResourceType>();418 assert(CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil &&419 "Only DXIL currently implements load with status");420 421 Intrinsic::ID IntrID = RT->getAttrs().RawBuffer422 ? llvm::Intrinsic::dx_resource_load_rawbuffer423 : llvm::Intrinsic::dx_resource_load_typedbuffer;424 425 llvm::Type *DataTy = ConvertType(E->getType());426 llvm::Type *RetTy = llvm::StructType::get(Builder.getContext(),427 {DataTy, Builder.getInt1Ty()});428 429 SmallVector<Value *, 3> Args;430 Args.push_back(HandleOp);431 Args.push_back(IndexOp);432 433 if (RT->getAttrs().RawBuffer) {434 Value *Offset = Builder.getInt32(0);435 Args.push_back(Offset);436 }437 438 // The load intrinsics give us a (T value, i1 status) pair -439 // shepherd these into the return value and out reference respectively.440 Value *ResRet =441 Builder.CreateIntrinsic(RetTy, IntrID, Args, {}, "ld.struct");442 Value *LoadedValue = Builder.CreateExtractValue(ResRet, {0}, "ld.value");443 Value *StatusBit = Builder.CreateExtractValue(ResRet, {1}, "ld.status");444 Value *ExtendedStatus =445 Builder.CreateZExt(StatusBit, Builder.getInt32Ty(), "ld.status.ext");446 Builder.CreateStore(ExtendedStatus, StatusAddr);447 448 return LoadedValue;449 }450 case Builtin::BI__builtin_hlsl_resource_uninitializedhandle: {451 llvm::Type *HandleTy = CGM.getTypes().ConvertType(E->getType());452 return llvm::PoisonValue::get(HandleTy);453 }454 case Builtin::BI__builtin_hlsl_resource_handlefrombinding: {455 llvm::Type *HandleTy = CGM.getTypes().ConvertType(E->getType());456 Value *RegisterOp = EmitScalarExpr(E->getArg(1));457 Value *SpaceOp = EmitScalarExpr(E->getArg(2));458 Value *RangeOp = EmitScalarExpr(E->getArg(3));459 Value *IndexOp = EmitScalarExpr(E->getArg(4));460 Value *Name = EmitScalarExpr(E->getArg(5));461 llvm::Intrinsic::ID IntrinsicID =462 CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic();463 SmallVector<Value *> Args{SpaceOp, RegisterOp, RangeOp, IndexOp, Name};464 return Builder.CreateIntrinsic(HandleTy, IntrinsicID, Args);465 }466 case Builtin::BI__builtin_hlsl_resource_handlefromimplicitbinding: {467 llvm::Type *HandleTy = CGM.getTypes().ConvertType(E->getType());468 Value *OrderID = EmitScalarExpr(E->getArg(1));469 Value *SpaceOp = EmitScalarExpr(E->getArg(2));470 Value *RangeOp = EmitScalarExpr(E->getArg(3));471 Value *IndexOp = EmitScalarExpr(E->getArg(4));472 Value *Name = EmitScalarExpr(E->getArg(5));473 llvm::Intrinsic::ID IntrinsicID =474 CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();475 SmallVector<Value *> Args{OrderID, SpaceOp, RangeOp, IndexOp, Name};476 return Builder.CreateIntrinsic(HandleTy, IntrinsicID, Args);477 }478 case Builtin::BI__builtin_hlsl_resource_counterhandlefromimplicitbinding: {479 Value *MainHandle = EmitScalarExpr(E->getArg(0));480 if (!CGM.getTriple().isSPIRV())481 return MainHandle;482 483 llvm::Type *HandleTy = CGM.getTypes().ConvertType(E->getType());484 Value *OrderID = EmitScalarExpr(E->getArg(1));485 Value *SpaceOp = EmitScalarExpr(E->getArg(2));486 llvm::Intrinsic::ID IntrinsicID =487 llvm::Intrinsic::spv_resource_counterhandlefromimplicitbinding;488 SmallVector<Value *> Args{MainHandle, OrderID, SpaceOp};489 return Builder.CreateIntrinsic(HandleTy, IntrinsicID, Args);490 }491 case Builtin::BI__builtin_hlsl_resource_nonuniformindex: {492 Value *IndexOp = EmitScalarExpr(E->getArg(0));493 llvm::Type *RetTy = ConvertType(E->getType());494 return Builder.CreateIntrinsic(495 RetTy, CGM.getHLSLRuntime().getNonUniformResourceIndexIntrinsic(),496 ArrayRef<Value *>{IndexOp});497 }498 case Builtin::BI__builtin_hlsl_resource_getdimensions_x: {499 Value *Handle = EmitScalarExpr(E->getArg(0));500 LValue Dim = EmitLValue(E->getArg(1));501 llvm::Type *RetTy = llvm::Type::getInt32Ty(getLLVMContext());502 Value *DimValue = Builder.CreateIntrinsic(503 RetTy, CGM.getHLSLRuntime().getGetDimensionsXIntrinsic(),504 ArrayRef<Value *>{Handle});505 return Builder.CreateStore(DimValue, Dim.getAddress());506 }507 case Builtin::BI__builtin_hlsl_resource_getstride: {508 LValue Stride = EmitLValue(E->getArg(1));509 return emitBufferStride(this, E->getArg(0), Stride);510 }511 case Builtin::BI__builtin_hlsl_all: {512 Value *Op0 = EmitScalarExpr(E->getArg(0));513 return Builder.CreateIntrinsic(514 /*ReturnType=*/llvm::Type::getInt1Ty(getLLVMContext()),515 CGM.getHLSLRuntime().getAllIntrinsic(), ArrayRef<Value *>{Op0}, nullptr,516 "hlsl.all");517 }518 case Builtin::BI__builtin_hlsl_and: {519 Value *Op0 = EmitScalarExpr(E->getArg(0));520 Value *Op1 = EmitScalarExpr(E->getArg(1));521 return Builder.CreateAnd(Op0, Op1, "hlsl.and");522 }523 case Builtin::BI__builtin_hlsl_or: {524 Value *Op0 = EmitScalarExpr(E->getArg(0));525 Value *Op1 = EmitScalarExpr(E->getArg(1));526 return Builder.CreateOr(Op0, Op1, "hlsl.or");527 }528 case Builtin::BI__builtin_hlsl_any: {529 Value *Op0 = EmitScalarExpr(E->getArg(0));530 return Builder.CreateIntrinsic(531 /*ReturnType=*/llvm::Type::getInt1Ty(getLLVMContext()),532 CGM.getHLSLRuntime().getAnyIntrinsic(), ArrayRef<Value *>{Op0}, nullptr,533 "hlsl.any");534 }535 case Builtin::BI__builtin_hlsl_asdouble:536 return handleAsDoubleBuiltin(*this, E);537 case Builtin::BI__builtin_hlsl_elementwise_clamp: {538 Value *OpX = EmitScalarExpr(E->getArg(0));539 Value *OpMin = EmitScalarExpr(E->getArg(1));540 Value *OpMax = EmitScalarExpr(E->getArg(2));541 542 QualType Ty = E->getArg(0)->getType();543 if (auto *VecTy = Ty->getAs<VectorType>())544 Ty = VecTy->getElementType();545 546 Intrinsic::ID Intr;547 if (Ty->isFloatingType()) {548 Intr = CGM.getHLSLRuntime().getNClampIntrinsic();549 } else if (Ty->isUnsignedIntegerType()) {550 Intr = CGM.getHLSLRuntime().getUClampIntrinsic();551 } else {552 assert(Ty->isSignedIntegerType());553 Intr = CGM.getHLSLRuntime().getSClampIntrinsic();554 }555 return Builder.CreateIntrinsic(556 /*ReturnType=*/OpX->getType(), Intr,557 ArrayRef<Value *>{OpX, OpMin, OpMax}, nullptr, "hlsl.clamp");558 }559 case Builtin::BI__builtin_hlsl_crossf16:560 case Builtin::BI__builtin_hlsl_crossf32: {561 Value *Op0 = EmitScalarExpr(E->getArg(0));562 Value *Op1 = EmitScalarExpr(E->getArg(1));563 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&564 E->getArg(1)->getType()->hasFloatingRepresentation() &&565 "cross operands must have a float representation");566 // make sure each vector has exactly 3 elements567 assert(568 E->getArg(0)->getType()->castAs<VectorType>()->getNumElements() == 3 &&569 E->getArg(1)->getType()->castAs<VectorType>()->getNumElements() == 3 &&570 "input vectors must have 3 elements each");571 return Builder.CreateIntrinsic(572 /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getCrossIntrinsic(),573 ArrayRef<Value *>{Op0, Op1}, nullptr, "hlsl.cross");574 }575 case Builtin::BI__builtin_hlsl_dot: {576 Value *Op0 = EmitScalarExpr(E->getArg(0));577 Value *Op1 = EmitScalarExpr(E->getArg(1));578 llvm::Type *T0 = Op0->getType();579 llvm::Type *T1 = Op1->getType();580 581 // If the arguments are scalars, just emit a multiply582 if (!T0->isVectorTy() && !T1->isVectorTy()) {583 if (T0->isFloatingPointTy())584 return Builder.CreateFMul(Op0, Op1, "hlsl.dot");585 586 if (T0->isIntegerTy())587 return Builder.CreateMul(Op0, Op1, "hlsl.dot");588 589 llvm_unreachable(590 "Scalar dot product is only supported on ints and floats.");591 }592 // For vectors, validate types and emit the appropriate intrinsic593 assert(CGM.getContext().hasSameUnqualifiedType(E->getArg(0)->getType(),594 E->getArg(1)->getType()) &&595 "Dot product operands must have the same type.");596 597 auto *VecTy0 = E->getArg(0)->getType()->castAs<VectorType>();598 assert(VecTy0 && "Dot product argument must be a vector.");599 600 return Builder.CreateIntrinsic(601 /*ReturnType=*/T0->getScalarType(),602 getDotProductIntrinsic(CGM.getHLSLRuntime(), VecTy0->getElementType()),603 ArrayRef<Value *>{Op0, Op1}, nullptr, "hlsl.dot");604 }605 case Builtin::BI__builtin_hlsl_dot4add_i8packed: {606 Value *X = EmitScalarExpr(E->getArg(0));607 Value *Y = EmitScalarExpr(E->getArg(1));608 Value *Acc = EmitScalarExpr(E->getArg(2));609 610 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();611 // Note that the argument order disagrees between the builtin and the612 // intrinsic here.613 return Builder.CreateIntrinsic(614 /*ReturnType=*/Acc->getType(), ID, ArrayRef<Value *>{Acc, X, Y},615 nullptr, "hlsl.dot4add.i8packed");616 }617 case Builtin::BI__builtin_hlsl_dot4add_u8packed: {618 Value *X = EmitScalarExpr(E->getArg(0));619 Value *Y = EmitScalarExpr(E->getArg(1));620 Value *Acc = EmitScalarExpr(E->getArg(2));621 622 Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();623 // Note that the argument order disagrees between the builtin and the624 // intrinsic here.625 return Builder.CreateIntrinsic(626 /*ReturnType=*/Acc->getType(), ID, ArrayRef<Value *>{Acc, X, Y},627 nullptr, "hlsl.dot4add.u8packed");628 }629 case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {630 Value *X = EmitScalarExpr(E->getArg(0));631 632 return Builder.CreateIntrinsic(633 /*ReturnType=*/ConvertType(E->getType()),634 getFirstBitHighIntrinsic(CGM.getHLSLRuntime(), E->getArg(0)->getType()),635 ArrayRef<Value *>{X}, nullptr, "hlsl.firstbithigh");636 }637 case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {638 Value *X = EmitScalarExpr(E->getArg(0));639 640 return Builder.CreateIntrinsic(641 /*ReturnType=*/ConvertType(E->getType()),642 CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), ArrayRef<Value *>{X},643 nullptr, "hlsl.firstbitlow");644 }645 case Builtin::BI__builtin_hlsl_lerp: {646 Value *X = EmitScalarExpr(E->getArg(0));647 Value *Y = EmitScalarExpr(E->getArg(1));648 Value *S = EmitScalarExpr(E->getArg(2));649 if (!E->getArg(0)->getType()->hasFloatingRepresentation())650 llvm_unreachable("lerp operand must have a float representation");651 return Builder.CreateIntrinsic(652 /*ReturnType=*/X->getType(), CGM.getHLSLRuntime().getLerpIntrinsic(),653 ArrayRef<Value *>{X, Y, S}, nullptr, "hlsl.lerp");654 }655 case Builtin::BI__builtin_hlsl_normalize: {656 Value *X = EmitScalarExpr(E->getArg(0));657 658 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&659 "normalize operand must have a float representation");660 661 return Builder.CreateIntrinsic(662 /*ReturnType=*/X->getType(),663 CGM.getHLSLRuntime().getNormalizeIntrinsic(), ArrayRef<Value *>{X},664 nullptr, "hlsl.normalize");665 }666 case Builtin::BI__builtin_hlsl_elementwise_degrees: {667 Value *X = EmitScalarExpr(E->getArg(0));668 669 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&670 "degree operand must have a float representation");671 672 return Builder.CreateIntrinsic(673 /*ReturnType=*/X->getType(), CGM.getHLSLRuntime().getDegreesIntrinsic(),674 ArrayRef<Value *>{X}, nullptr, "hlsl.degrees");675 }676 case Builtin::BI__builtin_hlsl_elementwise_f16tof32: {677 return handleElementwiseF16ToF32(*this, E);678 }679 case Builtin::BI__builtin_hlsl_elementwise_frac: {680 Value *Op0 = EmitScalarExpr(E->getArg(0));681 if (!E->getArg(0)->getType()->hasFloatingRepresentation())682 llvm_unreachable("frac operand must have a float representation");683 return Builder.CreateIntrinsic(684 /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getFracIntrinsic(),685 ArrayRef<Value *>{Op0}, nullptr, "hlsl.frac");686 }687 case Builtin::BI__builtin_hlsl_elementwise_isinf: {688 Value *Op0 = EmitScalarExpr(E->getArg(0));689 llvm::Type *Xty = Op0->getType();690 llvm::Type *retType = llvm::Type::getInt1Ty(this->getLLVMContext());691 if (Xty->isVectorTy()) {692 auto *XVecTy = E->getArg(0)->getType()->castAs<VectorType>();693 retType = llvm::VectorType::get(694 retType, ElementCount::getFixed(XVecTy->getNumElements()));695 }696 if (!E->getArg(0)->getType()->hasFloatingRepresentation())697 llvm_unreachable("isinf operand must have a float representation");698 return Builder.CreateIntrinsic(699 retType, CGM.getHLSLRuntime().getIsInfIntrinsic(),700 ArrayRef<Value *>{Op0}, nullptr, "hlsl.isinf");701 }702 case Builtin::BI__builtin_hlsl_elementwise_isnan: {703 Value *Op0 = EmitScalarExpr(E->getArg(0));704 llvm::Type *Xty = Op0->getType();705 llvm::Type *retType = llvm::Type::getInt1Ty(this->getLLVMContext());706 if (Xty->isVectorTy()) {707 auto *XVecTy = E->getArg(0)->getType()->castAs<VectorType>();708 retType = llvm::VectorType::get(709 retType, ElementCount::getFixed(XVecTy->getNumElements()));710 }711 if (!E->getArg(0)->getType()->hasFloatingRepresentation())712 llvm_unreachable("isnan operand must have a float representation");713 return Builder.CreateIntrinsic(714 retType, CGM.getHLSLRuntime().getIsNaNIntrinsic(),715 ArrayRef<Value *>{Op0}, nullptr, "hlsl.isnan");716 }717 case Builtin::BI__builtin_hlsl_mad: {718 Value *M = EmitScalarExpr(E->getArg(0));719 Value *A = EmitScalarExpr(E->getArg(1));720 Value *B = EmitScalarExpr(E->getArg(2));721 if (E->getArg(0)->getType()->hasFloatingRepresentation())722 return Builder.CreateIntrinsic(723 /*ReturnType*/ M->getType(), Intrinsic::fmuladd,724 ArrayRef<Value *>{M, A, B}, nullptr, "hlsl.fmad");725 726 if (E->getArg(0)->getType()->hasSignedIntegerRepresentation()) {727 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)728 return Builder.CreateIntrinsic(729 /*ReturnType*/ M->getType(), Intrinsic::dx_imad,730 ArrayRef<Value *>{M, A, B}, nullptr, "dx.imad");731 732 Value *Mul = Builder.CreateNSWMul(M, A);733 return Builder.CreateNSWAdd(Mul, B);734 }735 assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());736 if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)737 return Builder.CreateIntrinsic(738 /*ReturnType=*/M->getType(), Intrinsic::dx_umad,739 ArrayRef<Value *>{M, A, B}, nullptr, "dx.umad");740 741 Value *Mul = Builder.CreateNUWMul(M, A);742 return Builder.CreateNUWAdd(Mul, B);743 }744 case Builtin::BI__builtin_hlsl_elementwise_rcp: {745 Value *Op0 = EmitScalarExpr(E->getArg(0));746 if (!E->getArg(0)->getType()->hasFloatingRepresentation())747 llvm_unreachable("rcp operand must have a float representation");748 llvm::Type *Ty = Op0->getType();749 llvm::Type *EltTy = Ty->getScalarType();750 Constant *One = Ty->isVectorTy()751 ? ConstantVector::getSplat(752 ElementCount::getFixed(753 cast<FixedVectorType>(Ty)->getNumElements()),754 ConstantFP::get(EltTy, 1.0))755 : ConstantFP::get(EltTy, 1.0);756 return Builder.CreateFDiv(One, Op0, "hlsl.rcp");757 }758 case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {759 Value *Op0 = EmitScalarExpr(E->getArg(0));760 if (!E->getArg(0)->getType()->hasFloatingRepresentation())761 llvm_unreachable("rsqrt operand must have a float representation");762 return Builder.CreateIntrinsic(763 /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getRsqrtIntrinsic(),764 ArrayRef<Value *>{Op0}, nullptr, "hlsl.rsqrt");765 }766 case Builtin::BI__builtin_hlsl_elementwise_saturate: {767 Value *Op0 = EmitScalarExpr(E->getArg(0));768 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&769 "saturate operand must have a float representation");770 return Builder.CreateIntrinsic(771 /*ReturnType=*/Op0->getType(),772 CGM.getHLSLRuntime().getSaturateIntrinsic(), ArrayRef<Value *>{Op0},773 nullptr, "hlsl.saturate");774 }775 case Builtin::BI__builtin_hlsl_select: {776 Value *OpCond = EmitScalarExpr(E->getArg(0));777 RValue RValTrue = EmitAnyExpr(E->getArg(1));778 Value *OpTrue =779 RValTrue.isScalar()780 ? RValTrue.getScalarVal()781 : Builder.CreateLoad(RValTrue.getAggregateAddress(), "true_val");782 RValue RValFalse = EmitAnyExpr(E->getArg(2));783 Value *OpFalse =784 RValFalse.isScalar()785 ? RValFalse.getScalarVal()786 : Builder.CreateLoad(RValFalse.getAggregateAddress(), "false_val");787 if (auto *VTy = E->getType()->getAs<VectorType>()) {788 if (!OpTrue->getType()->isVectorTy())789 OpTrue =790 Builder.CreateVectorSplat(VTy->getNumElements(), OpTrue, "splat");791 if (!OpFalse->getType()->isVectorTy())792 OpFalse =793 Builder.CreateVectorSplat(VTy->getNumElements(), OpFalse, "splat");794 }795 796 Value *SelectVal =797 Builder.CreateSelect(OpCond, OpTrue, OpFalse, "hlsl.select");798 if (!RValTrue.isScalar())799 Builder.CreateStore(SelectVal, ReturnValue.getAddress(),800 ReturnValue.isVolatile());801 802 return SelectVal;803 }804 case Builtin::BI__builtin_hlsl_step: {805 Value *Op0 = EmitScalarExpr(E->getArg(0));806 Value *Op1 = EmitScalarExpr(E->getArg(1));807 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&808 E->getArg(1)->getType()->hasFloatingRepresentation() &&809 "step operands must have a float representation");810 return Builder.CreateIntrinsic(811 /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getStepIntrinsic(),812 ArrayRef<Value *>{Op0, Op1}, nullptr, "hlsl.step");813 }814 case Builtin::BI__builtin_hlsl_wave_active_all_true: {815 Value *Op = EmitScalarExpr(E->getArg(0));816 assert(Op->getType()->isIntegerTy(1) &&817 "Intrinsic WaveActiveAllTrue operand must be a bool");818 819 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();820 return EmitRuntimeCall(821 Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID), {Op});822 }823 case Builtin::BI__builtin_hlsl_wave_active_any_true: {824 Value *Op = EmitScalarExpr(E->getArg(0));825 assert(Op->getType()->isIntegerTy(1) &&826 "Intrinsic WaveActiveAnyTrue operand must be a bool");827 828 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();829 return EmitRuntimeCall(830 Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID), {Op});831 }832 case Builtin::BI__builtin_hlsl_wave_active_count_bits: {833 Value *OpExpr = EmitScalarExpr(E->getArg(0));834 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();835 return EmitRuntimeCall(836 Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID),837 ArrayRef{OpExpr});838 }839 case Builtin::BI__builtin_hlsl_wave_active_sum: {840 // Due to the use of variadic arguments, explicitly retreive argument841 Value *OpExpr = EmitScalarExpr(E->getArg(0));842 Intrinsic::ID IID = getWaveActiveSumIntrinsic(843 getTarget().getTriple().getArch(), CGM.getHLSLRuntime(),844 E->getArg(0)->getType());845 846 return EmitRuntimeCall(Intrinsic::getOrInsertDeclaration(847 &CGM.getModule(), IID, {OpExpr->getType()}),848 ArrayRef{OpExpr}, "hlsl.wave.active.sum");849 }850 case Builtin::BI__builtin_hlsl_wave_active_max: {851 // Due to the use of variadic arguments, explicitly retreive argument852 Value *OpExpr = EmitScalarExpr(E->getArg(0));853 Intrinsic::ID IID = getWaveActiveMaxIntrinsic(854 getTarget().getTriple().getArch(), CGM.getHLSLRuntime(),855 E->getArg(0)->getType());856 857 return EmitRuntimeCall(Intrinsic::getOrInsertDeclaration(858 &CGM.getModule(), IID, {OpExpr->getType()}),859 ArrayRef{OpExpr}, "hlsl.wave.active.max");860 }861 case Builtin::BI__builtin_hlsl_wave_active_min: {862 // Due to the use of variadic arguments, explicitly retreive argument863 Value *OpExpr = EmitScalarExpr(E->getArg(0));864 Intrinsic::ID IID = getWaveActiveMinIntrinsic(865 getTarget().getTriple().getArch(), CGM.getHLSLRuntime(),866 E->getArg(0)->getType());867 868 return EmitRuntimeCall(Intrinsic::getOrInsertDeclaration(869 &CGM.getModule(), IID, {OpExpr->getType()}),870 ArrayRef{OpExpr}, "hlsl.wave.active.min");871 }872 case Builtin::BI__builtin_hlsl_wave_get_lane_index: {873 // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in874 // defined in SPIRVBuiltins.td. So instead we manually get the matching name875 // for the DirectX intrinsic and the demangled builtin name876 switch (CGM.getTarget().getTriple().getArch()) {877 case llvm::Triple::dxil:878 return EmitRuntimeCall(Intrinsic::getOrInsertDeclaration(879 &CGM.getModule(), Intrinsic::dx_wave_getlaneindex));880 case llvm::Triple::spirv:881 return EmitRuntimeCall(CGM.CreateRuntimeFunction(882 llvm::FunctionType::get(IntTy, {}, false),883 "__hlsl_wave_get_lane_index", {}, false, true));884 default:885 llvm_unreachable(886 "Intrinsic WaveGetLaneIndex not supported by target architecture");887 }888 }889 case Builtin::BI__builtin_hlsl_wave_is_first_lane: {890 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();891 return EmitRuntimeCall(892 Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));893 }894 case Builtin::BI__builtin_hlsl_wave_get_lane_count: {895 Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveGetLaneCountIntrinsic();896 return EmitRuntimeCall(897 Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));898 }899 case Builtin::BI__builtin_hlsl_wave_read_lane_at: {900 // Due to the use of variadic arguments we must explicitly retreive them and901 // create our function type.902 Value *OpExpr = EmitScalarExpr(E->getArg(0));903 Value *OpIndex = EmitScalarExpr(E->getArg(1));904 return EmitRuntimeCall(905 Intrinsic::getOrInsertDeclaration(906 &CGM.getModule(), CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),907 {OpExpr->getType()}),908 ArrayRef{OpExpr, OpIndex}, "hlsl.wave.readlane");909 }910 case Builtin::BI__builtin_hlsl_elementwise_sign: {911 auto *Arg0 = E->getArg(0);912 Value *Op0 = EmitScalarExpr(Arg0);913 llvm::Type *Xty = Op0->getType();914 llvm::Type *retType = llvm::Type::getInt32Ty(this->getLLVMContext());915 if (Xty->isVectorTy()) {916 auto *XVecTy = Arg0->getType()->castAs<VectorType>();917 retType = llvm::VectorType::get(918 retType, ElementCount::getFixed(XVecTy->getNumElements()));919 }920 assert((Arg0->getType()->hasFloatingRepresentation() ||921 Arg0->getType()->hasIntegerRepresentation()) &&922 "sign operand must have a float or int representation");923 924 if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {925 Value *Cmp = Builder.CreateICmpEQ(Op0, ConstantInt::get(Xty, 0));926 return Builder.CreateSelect(Cmp, ConstantInt::get(retType, 0),927 ConstantInt::get(retType, 1), "hlsl.sign");928 }929 930 return Builder.CreateIntrinsic(931 retType, CGM.getHLSLRuntime().getSignIntrinsic(),932 ArrayRef<Value *>{Op0}, nullptr, "hlsl.sign");933 }934 case Builtin::BI__builtin_hlsl_elementwise_radians: {935 Value *Op0 = EmitScalarExpr(E->getArg(0));936 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&937 "radians operand must have a float representation");938 return Builder.CreateIntrinsic(939 /*ReturnType=*/Op0->getType(),940 CGM.getHLSLRuntime().getRadiansIntrinsic(), ArrayRef<Value *>{Op0},941 nullptr, "hlsl.radians");942 }943 case Builtin::BI__builtin_hlsl_buffer_update_counter: {944 Value *ResHandle = EmitScalarExpr(E->getArg(0));945 Value *Offset = EmitScalarExpr(E->getArg(1));946 Value *OffsetI8 = Builder.CreateIntCast(Offset, Int8Ty, true);947 return Builder.CreateIntrinsic(948 /*ReturnType=*/Offset->getType(),949 CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),950 ArrayRef<Value *>{ResHandle, OffsetI8}, nullptr);951 }952 case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {953 954 assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&955 E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&956 E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&957 "asuint operands types mismatch");958 return handleHlslSplitdouble(E, this);959 }960 case Builtin::BI__builtin_hlsl_elementwise_clip:961 assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&962 "clip operands types mismatch");963 return handleHlslClip(E, this);964 case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {965 Intrinsic::ID ID =966 CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();967 return EmitRuntimeCall(968 Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));969 }970 case Builtin::BI__builtin_hlsl_elementwise_ddx_coarse: {971 Value *Op0 = EmitScalarExpr(E->getArg(0));972 if (!E->getArg(0)->getType()->hasFloatingRepresentation())973 llvm_unreachable("ddx_coarse operand must have a float representation");974 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdxCoarseIntrinsic();975 return Builder.CreateIntrinsic(/*ReturnType=*/Op0->getType(), ID,976 ArrayRef<Value *>{Op0}, nullptr,977 "hlsl.ddx.coarse");978 }979 case Builtin::BI__builtin_hlsl_elementwise_ddy_coarse: {980 Value *Op0 = EmitScalarExpr(E->getArg(0));981 if (!E->getArg(0)->getType()->hasFloatingRepresentation())982 llvm_unreachable("ddy_coarse operand must have a float representation");983 Intrinsic::ID ID = CGM.getHLSLRuntime().getDdyCoarseIntrinsic();984 return Builder.CreateIntrinsic(/*ReturnType=*/Op0->getType(), ID,985 ArrayRef<Value *>{Op0}, nullptr,986 "hlsl.ddy.coarse");987 }988 case Builtin::BI__builtin_get_spirv_spec_constant_bool:989 case Builtin::BI__builtin_get_spirv_spec_constant_short:990 case Builtin::BI__builtin_get_spirv_spec_constant_ushort:991 case Builtin::BI__builtin_get_spirv_spec_constant_int:992 case Builtin::BI__builtin_get_spirv_spec_constant_uint:993 case Builtin::BI__builtin_get_spirv_spec_constant_longlong:994 case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:995 case Builtin::BI__builtin_get_spirv_spec_constant_half:996 case Builtin::BI__builtin_get_spirv_spec_constant_float:997 case Builtin::BI__builtin_get_spirv_spec_constant_double: {998 llvm::Function *SpecConstantFn = getSpecConstantFunction(E->getType());999 llvm::Value *SpecId = EmitScalarExpr(E->getArg(0));1000 llvm::Value *DefaultVal = EmitScalarExpr(E->getArg(1));1001 llvm::Value *Args[] = {SpecId, DefaultVal};1002 return Builder.CreateCall(SpecConstantFn, Args);1003 }1004 }1005 return nullptr;1006}1007 1008llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(1009 const clang::QualType &SpecConstantType) {1010 1011 // Find or create the declaration for the function.1012 llvm::Module *M = &CGM.getModule();1013 std::string MangledName =1014 getSpecConstantFunctionName(SpecConstantType, getContext());1015 llvm::Function *SpecConstantFn = M->getFunction(MangledName);1016 1017 if (!SpecConstantFn) {1018 llvm::Type *IntType = ConvertType(getContext().IntTy);1019 llvm::Type *RetTy = ConvertType(SpecConstantType);1020 llvm::Type *ArgTypes[] = {IntType, RetTy};1021 llvm::FunctionType *FnTy = llvm::FunctionType::get(RetTy, ArgTypes, false);1022 SpecConstantFn = llvm::Function::Create(1023 FnTy, llvm::GlobalValue::ExternalLinkage, MangledName, M);1024 }1025 return SpecConstantFn;1026}1027