1127 lines · cpp
1//===- DXILIntrinsicExpansion.cpp - Prepare LLVM Module for DXIL encoding--===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8///9/// \file This file contains DXIL intrinsic expansions for those that don't have10// opcodes in DirectX Intermediate Language (DXIL).11//===----------------------------------------------------------------------===//12 13#include "DXILIntrinsicExpansion.h"14#include "DirectX.h"15#include "llvm/ADT/STLExtras.h"16#include "llvm/ADT/SmallVector.h"17#include "llvm/CodeGen/Passes.h"18#include "llvm/IR/IRBuilder.h"19#include "llvm/IR/InstrTypes.h"20#include "llvm/IR/Instruction.h"21#include "llvm/IR/Instructions.h"22#include "llvm/IR/Intrinsics.h"23#include "llvm/IR/IntrinsicsDirectX.h"24#include "llvm/IR/Module.h"25#include "llvm/IR/PassManager.h"26#include "llvm/IR/Type.h"27#include "llvm/Pass.h"28#include "llvm/Support/Casting.h"29#include "llvm/Support/ErrorHandling.h"30#include "llvm/Support/MathExtras.h"31 32#define DEBUG_TYPE "dxil-intrinsic-expansion"33 34using namespace llvm;35 36class DXILIntrinsicExpansionLegacy : public ModulePass {37 38public:39 bool runOnModule(Module &M) override;40 DXILIntrinsicExpansionLegacy() : ModulePass(ID) {}41 42 static char ID; // Pass identification.43};44 45static bool resourceAccessNeeds64BitExpansion(Module *M, Type *OverloadTy,46 bool IsRaw) {47 if (IsRaw && M->getTargetTriple().getDXILVersion() > VersionTuple(1, 2))48 return false;49 50 Type *ScalarTy = OverloadTy->getScalarType();51 return ScalarTy->isDoubleTy() || ScalarTy->isIntegerTy(64);52}53 54static Value *expand16BitIsInf(CallInst *Orig) {55 Module *M = Orig->getModule();56 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))57 return nullptr;58 59 Value *Val = Orig->getOperand(0);60 Type *ValTy = Val->getType();61 if (!ValTy->getScalarType()->isHalfTy())62 return nullptr;63 64 IRBuilder<> Builder(Orig);65 Type *IType = Type::getInt16Ty(M->getContext());66 Constant *PosInf =67 ValTy->isVectorTy()68 ? ConstantVector::getSplat(69 ElementCount::getFixed(70 cast<FixedVectorType>(ValTy)->getNumElements()),71 ConstantInt::get(IType, 0x7c00))72 : ConstantInt::get(IType, 0x7c00);73 74 Constant *NegInf =75 ValTy->isVectorTy()76 ? ConstantVector::getSplat(77 ElementCount::getFixed(78 cast<FixedVectorType>(ValTy)->getNumElements()),79 ConstantInt::get(IType, 0xfc00))80 : ConstantInt::get(IType, 0xfc00);81 82 Value *IVal = Builder.CreateBitCast(Val, PosInf->getType());83 Value *B1 = Builder.CreateICmpEQ(IVal, PosInf);84 Value *B2 = Builder.CreateICmpEQ(IVal, NegInf);85 Value *B3 = Builder.CreateOr(B1, B2);86 return B3;87}88 89static Value *expand16BitIsNaN(CallInst *Orig) {90 Module *M = Orig->getModule();91 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))92 return nullptr;93 94 Value *Val = Orig->getOperand(0);95 Type *ValTy = Val->getType();96 if (!ValTy->getScalarType()->isHalfTy())97 return nullptr;98 99 IRBuilder<> Builder(Orig);100 Type *IType = Type::getInt16Ty(M->getContext());101 102 Constant *ExpBitMask =103 ValTy->isVectorTy()104 ? ConstantVector::getSplat(105 ElementCount::getFixed(106 cast<FixedVectorType>(ValTy)->getNumElements()),107 ConstantInt::get(IType, 0x7c00))108 : ConstantInt::get(IType, 0x7c00);109 Constant *SigBitMask =110 ValTy->isVectorTy()111 ? ConstantVector::getSplat(112 ElementCount::getFixed(113 cast<FixedVectorType>(ValTy)->getNumElements()),114 ConstantInt::get(IType, 0x3ff))115 : ConstantInt::get(IType, 0x3ff);116 117 Constant *Zero =118 ValTy->isVectorTy()119 ? ConstantVector::getSplat(120 ElementCount::getFixed(121 cast<FixedVectorType>(ValTy)->getNumElements()),122 ConstantInt::get(IType, 0))123 : ConstantInt::get(IType, 0);124 125 Value *IVal = Builder.CreateBitCast(Val, ExpBitMask->getType());126 Value *Exp = Builder.CreateAnd(IVal, ExpBitMask);127 Value *B1 = Builder.CreateICmpEQ(Exp, ExpBitMask);128 129 Value *Sig = Builder.CreateAnd(IVal, SigBitMask);130 Value *B2 = Builder.CreateICmpNE(Sig, Zero);131 Value *B3 = Builder.CreateAnd(B1, B2);132 return B3;133}134 135static Value *expand16BitIsFinite(CallInst *Orig) {136 Module *M = Orig->getModule();137 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))138 return nullptr;139 140 Value *Val = Orig->getOperand(0);141 Type *ValTy = Val->getType();142 if (!ValTy->getScalarType()->isHalfTy())143 return nullptr;144 145 IRBuilder<> Builder(Orig);146 Type *IType = Type::getInt16Ty(M->getContext());147 148 Constant *ExpBitMask =149 ValTy->isVectorTy()150 ? ConstantVector::getSplat(151 ElementCount::getFixed(152 cast<FixedVectorType>(ValTy)->getNumElements()),153 ConstantInt::get(IType, 0x7c00))154 : ConstantInt::get(IType, 0x7c00);155 156 Value *IVal = Builder.CreateBitCast(Val, ExpBitMask->getType());157 Value *Exp = Builder.CreateAnd(IVal, ExpBitMask);158 Value *B1 = Builder.CreateICmpNE(Exp, ExpBitMask);159 return B1;160}161 162static Value *expand16BitIsNormal(CallInst *Orig) {163 Module *M = Orig->getModule();164 if (M->getTargetTriple().getDXILVersion() >= VersionTuple(1, 9))165 return nullptr;166 167 Value *Val = Orig->getOperand(0);168 Type *ValTy = Val->getType();169 if (!ValTy->getScalarType()->isHalfTy())170 return nullptr;171 172 IRBuilder<> Builder(Orig);173 Type *IType = Type::getInt16Ty(M->getContext());174 175 Constant *ExpBitMask =176 ValTy->isVectorTy()177 ? ConstantVector::getSplat(178 ElementCount::getFixed(179 cast<FixedVectorType>(ValTy)->getNumElements()),180 ConstantInt::get(IType, 0x7c00))181 : ConstantInt::get(IType, 0x7c00);182 Constant *Zero =183 ValTy->isVectorTy()184 ? ConstantVector::getSplat(185 ElementCount::getFixed(186 cast<FixedVectorType>(ValTy)->getNumElements()),187 ConstantInt::get(IType, 0))188 : ConstantInt::get(IType, 0);189 190 Value *IVal = Builder.CreateBitCast(Val, ExpBitMask->getType());191 Value *Exp = Builder.CreateAnd(IVal, ExpBitMask);192 Value *NotAllZeroes = Builder.CreateICmpNE(Exp, Zero);193 Value *NotAllOnes = Builder.CreateICmpNE(Exp, ExpBitMask);194 Value *B1 = Builder.CreateAnd(NotAllZeroes, NotAllOnes);195 return B1;196}197 198static bool isIntrinsicExpansion(Function &F) {199 switch (F.getIntrinsicID()) {200 case Intrinsic::assume:201 case Intrinsic::abs:202 case Intrinsic::atan2:203 case Intrinsic::exp:204 case Intrinsic::is_fpclass:205 case Intrinsic::log:206 case Intrinsic::log10:207 case Intrinsic::pow:208 case Intrinsic::powi:209 case Intrinsic::dx_all:210 case Intrinsic::dx_any:211 case Intrinsic::dx_cross:212 case Intrinsic::dx_uclamp:213 case Intrinsic::dx_sclamp:214 case Intrinsic::dx_nclamp:215 case Intrinsic::dx_degrees:216 case Intrinsic::dx_isinf:217 case Intrinsic::dx_isnan:218 case Intrinsic::dx_lerp:219 case Intrinsic::dx_normalize:220 case Intrinsic::dx_fdot:221 case Intrinsic::dx_sdot:222 case Intrinsic::dx_udot:223 case Intrinsic::dx_sign:224 case Intrinsic::dx_step:225 case Intrinsic::dx_radians:226 case Intrinsic::usub_sat:227 case Intrinsic::vector_reduce_add:228 case Intrinsic::vector_reduce_fadd:229 return true;230 case Intrinsic::dx_resource_load_rawbuffer:231 return resourceAccessNeeds64BitExpansion(232 F.getParent(), F.getReturnType()->getStructElementType(0),233 /*IsRaw*/ true);234 case Intrinsic::dx_resource_load_typedbuffer:235 return resourceAccessNeeds64BitExpansion(236 F.getParent(), F.getReturnType()->getStructElementType(0),237 /*IsRaw*/ false);238 case Intrinsic::dx_resource_store_rawbuffer:239 return resourceAccessNeeds64BitExpansion(240 F.getParent(), F.getFunctionType()->getParamType(3), /*IsRaw*/ true);241 case Intrinsic::dx_resource_store_typedbuffer:242 return resourceAccessNeeds64BitExpansion(243 F.getParent(), F.getFunctionType()->getParamType(2), /*IsRaw*/ false);244 }245 return false;246}247 248static Value *expandUsubSat(CallInst *Orig) {249 Value *A = Orig->getArgOperand(0);250 Value *B = Orig->getArgOperand(1);251 Type *Ty = A->getType();252 253 IRBuilder<> Builder(Orig);254 255 Value *Cmp = Builder.CreateICmpULT(A, B, "usub.cmp");256 Value *Sub = Builder.CreateSub(A, B, "usub.sub");257 Value *Zero = ConstantInt::get(Ty, 0);258 return Builder.CreateSelect(Cmp, Zero, Sub, "usub.sat");259}260 261static Value *expandVecReduceAdd(CallInst *Orig, Intrinsic::ID IntrinsicId) {262 assert(IntrinsicId == Intrinsic::vector_reduce_add ||263 IntrinsicId == Intrinsic::vector_reduce_fadd);264 265 IRBuilder<> Builder(Orig);266 bool IsFAdd = (IntrinsicId == Intrinsic::vector_reduce_fadd);267 268 Value *X = Orig->getOperand(IsFAdd ? 1 : 0);269 Type *Ty = X->getType();270 auto *XVec = dyn_cast<FixedVectorType>(Ty);271 unsigned XVecSize = XVec->getNumElements();272 Value *Sum = Builder.CreateExtractElement(X, static_cast<uint64_t>(0));273 274 // Handle the initial start value for floating-point addition.275 if (IsFAdd) {276 Constant *StartValue = dyn_cast<Constant>(Orig->getOperand(0));277 if (StartValue && !StartValue->isZeroValue())278 Sum = Builder.CreateFAdd(Sum, StartValue);279 }280 281 // Accumulate the remaining vector elements.282 for (unsigned I = 1; I < XVecSize; I++) {283 Value *Elt = Builder.CreateExtractElement(X, I);284 if (IsFAdd)285 Sum = Builder.CreateFAdd(Sum, Elt);286 else287 Sum = Builder.CreateAdd(Sum, Elt);288 }289 290 return Sum;291}292 293static Value *expandAbs(CallInst *Orig) {294 Value *X = Orig->getOperand(0);295 IRBuilder<> Builder(Orig);296 Type *Ty = X->getType();297 Type *EltTy = Ty->getScalarType();298 Constant *Zero = Ty->isVectorTy()299 ? ConstantVector::getSplat(300 ElementCount::getFixed(301 cast<FixedVectorType>(Ty)->getNumElements()),302 ConstantInt::get(EltTy, 0))303 : ConstantInt::get(EltTy, 0);304 auto *V = Builder.CreateSub(Zero, X);305 return Builder.CreateIntrinsic(Ty, Intrinsic::smax, {X, V}, nullptr,306 "dx.max");307}308 309static Value *expandCrossIntrinsic(CallInst *Orig) {310 311 VectorType *VT = cast<VectorType>(Orig->getType());312 if (cast<FixedVectorType>(VT)->getNumElements() != 3)313 reportFatalUsageError("return vector must have exactly 3 elements");314 315 Value *op0 = Orig->getOperand(0);316 Value *op1 = Orig->getOperand(1);317 IRBuilder<> Builder(Orig);318 319 Value *op0_x = Builder.CreateExtractElement(op0, (uint64_t)0, "x0");320 Value *op0_y = Builder.CreateExtractElement(op0, 1, "x1");321 Value *op0_z = Builder.CreateExtractElement(op0, 2, "x2");322 323 Value *op1_x = Builder.CreateExtractElement(op1, (uint64_t)0, "y0");324 Value *op1_y = Builder.CreateExtractElement(op1, 1, "y1");325 Value *op1_z = Builder.CreateExtractElement(op1, 2, "y2");326 327 auto MulSub = [&](Value *x0, Value *y0, Value *x1, Value *y1) -> Value * {328 Value *xy = Builder.CreateFMul(x0, y1);329 Value *yx = Builder.CreateFMul(y0, x1);330 return Builder.CreateFSub(xy, yx, Orig->getName());331 };332 333 Value *yz_zy = MulSub(op0_y, op0_z, op1_y, op1_z);334 Value *zx_xz = MulSub(op0_z, op0_x, op1_z, op1_x);335 Value *xy_yx = MulSub(op0_x, op0_y, op1_x, op1_y);336 337 Value *cross = PoisonValue::get(VT);338 cross = Builder.CreateInsertElement(cross, yz_zy, (uint64_t)0);339 cross = Builder.CreateInsertElement(cross, zx_xz, 1);340 cross = Builder.CreateInsertElement(cross, xy_yx, 2);341 return cross;342}343 344// Create appropriate DXIL float dot intrinsic for the given A and B operands345// The appropriate opcode will be determined by the size of the operands346// The dot product is placed in the position indicated by Orig347static Value *expandFloatDotIntrinsic(CallInst *Orig, Value *A, Value *B) {348 Type *ATy = A->getType();349 [[maybe_unused]] Type *BTy = B->getType();350 assert(ATy->isVectorTy() && BTy->isVectorTy());351 352 IRBuilder<> Builder(Orig);353 354 auto *AVec = dyn_cast<FixedVectorType>(ATy);355 356 assert(ATy->getScalarType()->isFloatingPointTy());357 358 Intrinsic::ID DotIntrinsic = Intrinsic::dx_dot4;359 int NumElts = AVec->getNumElements();360 switch (NumElts) {361 case 2:362 DotIntrinsic = Intrinsic::dx_dot2;363 break;364 case 3:365 DotIntrinsic = Intrinsic::dx_dot3;366 break;367 case 4:368 DotIntrinsic = Intrinsic::dx_dot4;369 break;370 default:371 reportFatalUsageError(372 "Invalid dot product input vector: length is outside 2-4");373 return nullptr;374 }375 376 SmallVector<Value *> Args;377 for (int I = 0; I < NumElts; ++I)378 Args.push_back(Builder.CreateExtractElement(A, Builder.getInt32(I)));379 for (int I = 0; I < NumElts; ++I)380 Args.push_back(Builder.CreateExtractElement(B, Builder.getInt32(I)));381 return Builder.CreateIntrinsic(ATy->getScalarType(), DotIntrinsic, Args,382 nullptr, "dot");383}384 385// Create the appropriate DXIL float dot intrinsic for the operands of Orig386// The appropriate opcode will be determined by the size of the operands387// The dot product is placed in the position indicated by Orig388static Value *expandFloatDotIntrinsic(CallInst *Orig) {389 return expandFloatDotIntrinsic(Orig, Orig->getOperand(0),390 Orig->getOperand(1));391}392 393// Expand integer dot product to multiply and add ops394static Value *expandIntegerDotIntrinsic(CallInst *Orig,395 Intrinsic::ID DotIntrinsic) {396 assert(DotIntrinsic == Intrinsic::dx_sdot ||397 DotIntrinsic == Intrinsic::dx_udot);398 Value *A = Orig->getOperand(0);399 Value *B = Orig->getOperand(1);400 Type *ATy = A->getType();401 [[maybe_unused]] Type *BTy = B->getType();402 assert(ATy->isVectorTy() && BTy->isVectorTy());403 404 IRBuilder<> Builder(Orig);405 406 auto *AVec = dyn_cast<FixedVectorType>(ATy);407 408 assert(ATy->getScalarType()->isIntegerTy());409 410 Value *Result;411 Intrinsic::ID MadIntrinsic = DotIntrinsic == Intrinsic::dx_sdot412 ? Intrinsic::dx_imad413 : Intrinsic::dx_umad;414 Value *Elt0 = Builder.CreateExtractElement(A, (uint64_t)0);415 Value *Elt1 = Builder.CreateExtractElement(B, (uint64_t)0);416 Result = Builder.CreateMul(Elt0, Elt1);417 for (unsigned I = 1; I < AVec->getNumElements(); I++) {418 Elt0 = Builder.CreateExtractElement(A, I);419 Elt1 = Builder.CreateExtractElement(B, I);420 Result = Builder.CreateIntrinsic(Result->getType(), MadIntrinsic,421 ArrayRef<Value *>{Elt0, Elt1, Result},422 nullptr, "dx.mad");423 }424 return Result;425}426 427static Value *expandExpIntrinsic(CallInst *Orig) {428 Value *X = Orig->getOperand(0);429 IRBuilder<> Builder(Orig);430 Type *Ty = X->getType();431 Type *EltTy = Ty->getScalarType();432 Constant *Log2eConst =433 Ty->isVectorTy() ? ConstantVector::getSplat(434 ElementCount::getFixed(435 cast<FixedVectorType>(Ty)->getNumElements()),436 ConstantFP::get(EltTy, numbers::log2ef))437 : ConstantFP::get(EltTy, numbers::log2ef);438 Value *NewX = Builder.CreateFMul(Log2eConst, X);439 auto *Exp2Call =440 Builder.CreateIntrinsic(Ty, Intrinsic::exp2, {NewX}, nullptr, "dx.exp2");441 Exp2Call->setTailCall(Orig->isTailCall());442 Exp2Call->setAttributes(Orig->getAttributes());443 return Exp2Call;444}445 446static Value *expandIsFPClass(CallInst *Orig) {447 Value *T = Orig->getArgOperand(1);448 auto *TCI = dyn_cast<ConstantInt>(T);449 450 // These FPClassTest cases have DXIL opcodes, so they will be handled in451 // DXIL Op Lowering instead for all non f16 cases.452 switch (TCI->getZExtValue()) {453 case FPClassTest::fcInf:454 return expand16BitIsInf(Orig);455 case FPClassTest::fcNan:456 return expand16BitIsNaN(Orig);457 case FPClassTest::fcNormal:458 return expand16BitIsNormal(Orig);459 case FPClassTest::fcFinite:460 return expand16BitIsFinite(Orig);461 }462 463 IRBuilder<> Builder(Orig);464 465 Value *F = Orig->getArgOperand(0);466 Type *FTy = F->getType();467 unsigned FNumElem = 0; // 0 => F is not a vector468 469 unsigned BitWidth; // Bit width of F or the ElemTy of F470 Type *BitCastTy; // An IntNTy of the same bitwidth as F or ElemTy of F471 472 if (auto *FVecTy = dyn_cast<FixedVectorType>(FTy)) {473 Type *ElemTy = FVecTy->getElementType();474 FNumElem = FVecTy->getNumElements();475 BitWidth = ElemTy->getPrimitiveSizeInBits();476 BitCastTy = FixedVectorType::get(Builder.getIntNTy(BitWidth), FNumElem);477 } else {478 BitWidth = FTy->getPrimitiveSizeInBits();479 BitCastTy = Builder.getIntNTy(BitWidth);480 }481 482 Value *FBitCast = Builder.CreateBitCast(F, BitCastTy);483 switch (TCI->getZExtValue()) {484 case FPClassTest::fcNegZero: {485 Value *NegZero =486 ConstantInt::get(Builder.getIntNTy(BitWidth), 1 << (BitWidth - 1));487 Value *RetVal;488 if (FNumElem) {489 Value *NegZeroSplat = Builder.CreateVectorSplat(FNumElem, NegZero);490 RetVal =491 Builder.CreateICmpEQ(FBitCast, NegZeroSplat, "is.fpclass.negzero");492 } else493 RetVal = Builder.CreateICmpEQ(FBitCast, NegZero, "is.fpclass.negzero");494 return RetVal;495 }496 default:497 reportFatalUsageError("Unsupported FPClassTest");498 }499}500 501static Value *expandAnyOrAllIntrinsic(CallInst *Orig,502 Intrinsic::ID IntrinsicId) {503 Value *X = Orig->getOperand(0);504 IRBuilder<> Builder(Orig);505 Type *Ty = X->getType();506 Type *EltTy = Ty->getScalarType();507 508 auto ApplyOp = [&Builder](Intrinsic::ID IntrinsicId, Value *Result,509 Value *Elt) {510 if (IntrinsicId == Intrinsic::dx_any)511 return Builder.CreateOr(Result, Elt);512 assert(IntrinsicId == Intrinsic::dx_all);513 return Builder.CreateAnd(Result, Elt);514 };515 516 Value *Result = nullptr;517 if (!Ty->isVectorTy()) {518 Result = EltTy->isFloatingPointTy()519 ? Builder.CreateFCmpUNE(X, ConstantFP::get(EltTy, 0))520 : Builder.CreateICmpNE(X, ConstantInt::get(EltTy, 0));521 } else {522 auto *XVec = dyn_cast<FixedVectorType>(Ty);523 Value *Cond =524 EltTy->isFloatingPointTy()525 ? Builder.CreateFCmpUNE(526 X, ConstantVector::getSplat(527 ElementCount::getFixed(XVec->getNumElements()),528 ConstantFP::get(EltTy, 0)))529 : Builder.CreateICmpNE(530 X, ConstantVector::getSplat(531 ElementCount::getFixed(XVec->getNumElements()),532 ConstantInt::get(EltTy, 0)));533 Result = Builder.CreateExtractElement(Cond, (uint64_t)0);534 for (unsigned I = 1; I < XVec->getNumElements(); I++) {535 Value *Elt = Builder.CreateExtractElement(Cond, I);536 Result = ApplyOp(IntrinsicId, Result, Elt);537 }538 }539 return Result;540}541 542static Value *expandLerpIntrinsic(CallInst *Orig) {543 Value *X = Orig->getOperand(0);544 Value *Y = Orig->getOperand(1);545 Value *S = Orig->getOperand(2);546 IRBuilder<> Builder(Orig);547 auto *V = Builder.CreateFSub(Y, X);548 V = Builder.CreateFMul(S, V);549 return Builder.CreateFAdd(X, V, "dx.lerp");550}551 552static Value *expandLogIntrinsic(CallInst *Orig,553 float LogConstVal = numbers::ln2f) {554 Value *X = Orig->getOperand(0);555 IRBuilder<> Builder(Orig);556 Type *Ty = X->getType();557 Type *EltTy = Ty->getScalarType();558 Constant *Ln2Const =559 Ty->isVectorTy() ? ConstantVector::getSplat(560 ElementCount::getFixed(561 cast<FixedVectorType>(Ty)->getNumElements()),562 ConstantFP::get(EltTy, LogConstVal))563 : ConstantFP::get(EltTy, LogConstVal);564 auto *Log2Call =565 Builder.CreateIntrinsic(Ty, Intrinsic::log2, {X}, nullptr, "elt.log2");566 Log2Call->setTailCall(Orig->isTailCall());567 Log2Call->setAttributes(Orig->getAttributes());568 return Builder.CreateFMul(Ln2Const, Log2Call);569}570static Value *expandLog10Intrinsic(CallInst *Orig) {571 return expandLogIntrinsic(Orig, numbers::ln2f / numbers::ln10f);572}573 574// Use dot product of vector operand with itself to calculate the length.575// Divide the vector by that length to normalize it.576static Value *expandNormalizeIntrinsic(CallInst *Orig) {577 Value *X = Orig->getOperand(0);578 Type *Ty = Orig->getType();579 Type *EltTy = Ty->getScalarType();580 IRBuilder<> Builder(Orig);581 582 auto *XVec = dyn_cast<FixedVectorType>(Ty);583 if (!XVec) {584 if (auto *constantFP = dyn_cast<ConstantFP>(X)) {585 const APFloat &fpVal = constantFP->getValueAPF();586 if (fpVal.isZero())587 reportFatalUsageError("Invalid input scalar: length is zero");588 }589 return Builder.CreateFDiv(X, X);590 }591 592 Value *DotProduct = expandFloatDotIntrinsic(Orig, X, X);593 594 // verify that the length is non-zero595 // (if the dot product is non-zero, then the length is non-zero)596 if (auto *constantFP = dyn_cast<ConstantFP>(DotProduct)) {597 const APFloat &fpVal = constantFP->getValueAPF();598 if (fpVal.isZero())599 reportFatalUsageError("Invalid input vector: length is zero");600 }601 602 Value *Multiplicand = Builder.CreateIntrinsic(EltTy, Intrinsic::dx_rsqrt,603 ArrayRef<Value *>{DotProduct},604 nullptr, "dx.rsqrt");605 606 Value *MultiplicandVec =607 Builder.CreateVectorSplat(XVec->getNumElements(), Multiplicand);608 return Builder.CreateFMul(X, MultiplicandVec);609}610 611static Value *expandAtan2Intrinsic(CallInst *Orig) {612 Value *Y = Orig->getOperand(0);613 Value *X = Orig->getOperand(1);614 Type *Ty = X->getType();615 IRBuilder<> Builder(Orig);616 Builder.setFastMathFlags(Orig->getFastMathFlags());617 618 Value *Tan = Builder.CreateFDiv(Y, X);619 620 CallInst *Atan =621 Builder.CreateIntrinsic(Ty, Intrinsic::atan, {Tan}, nullptr, "Elt.Atan");622 Atan->setTailCall(Orig->isTailCall());623 Atan->setAttributes(Orig->getAttributes());624 625 // Modify atan result based on https://en.wikipedia.org/wiki/Atan2.626 Constant *Pi = ConstantFP::get(Ty, llvm::numbers::pi);627 Constant *HalfPi = ConstantFP::get(Ty, llvm::numbers::pi / 2);628 Constant *NegHalfPi = ConstantFP::get(Ty, -llvm::numbers::pi / 2);629 Constant *Zero = ConstantFP::get(Ty, 0);630 Value *AtanAddPi = Builder.CreateFAdd(Atan, Pi);631 Value *AtanSubPi = Builder.CreateFSub(Atan, Pi);632 633 // x > 0 -> atan.634 Value *Result = Atan;635 Value *XLt0 = Builder.CreateFCmpOLT(X, Zero);636 Value *XEq0 = Builder.CreateFCmpOEQ(X, Zero);637 Value *YGe0 = Builder.CreateFCmpOGE(Y, Zero);638 Value *YLt0 = Builder.CreateFCmpOLT(Y, Zero);639 640 // x < 0, y >= 0 -> atan + pi.641 Value *XLt0AndYGe0 = Builder.CreateAnd(XLt0, YGe0);642 Result = Builder.CreateSelect(XLt0AndYGe0, AtanAddPi, Result);643 644 // x < 0, y < 0 -> atan - pi.645 Value *XLt0AndYLt0 = Builder.CreateAnd(XLt0, YLt0);646 Result = Builder.CreateSelect(XLt0AndYLt0, AtanSubPi, Result);647 648 // x == 0, y < 0 -> -pi/2649 Value *XEq0AndYLt0 = Builder.CreateAnd(XEq0, YLt0);650 Result = Builder.CreateSelect(XEq0AndYLt0, NegHalfPi, Result);651 652 // x == 0, y > 0 -> pi/2653 Value *XEq0AndYGe0 = Builder.CreateAnd(XEq0, YGe0);654 Result = Builder.CreateSelect(XEq0AndYGe0, HalfPi, Result);655 656 return Result;657}658 659static Value *expandPowIntrinsic(CallInst *Orig, Intrinsic::ID IntrinsicId) {660 661 Value *X = Orig->getOperand(0);662 Value *Y = Orig->getOperand(1);663 Type *Ty = X->getType();664 IRBuilder<> Builder(Orig);665 666 if (IntrinsicId == Intrinsic::powi)667 Y = Builder.CreateSIToFP(Y, Ty);668 669 auto *Log2Call =670 Builder.CreateIntrinsic(Ty, Intrinsic::log2, {X}, nullptr, "elt.log2");671 auto *Mul = Builder.CreateFMul(Log2Call, Y);672 auto *Exp2Call =673 Builder.CreateIntrinsic(Ty, Intrinsic::exp2, {Mul}, nullptr, "elt.exp2");674 Exp2Call->setTailCall(Orig->isTailCall());675 Exp2Call->setAttributes(Orig->getAttributes());676 return Exp2Call;677}678 679static Value *expandStepIntrinsic(CallInst *Orig) {680 681 Value *X = Orig->getOperand(0);682 Value *Y = Orig->getOperand(1);683 Type *Ty = X->getType();684 IRBuilder<> Builder(Orig);685 686 Constant *One = ConstantFP::get(Ty->getScalarType(), 1.0);687 Constant *Zero = ConstantFP::get(Ty->getScalarType(), 0.0);688 Value *Cond = Builder.CreateFCmpOLT(Y, X);689 690 if (Ty != Ty->getScalarType()) {691 auto *XVec = dyn_cast<FixedVectorType>(Ty);692 One = ConstantVector::getSplat(693 ElementCount::getFixed(XVec->getNumElements()), One);694 Zero = ConstantVector::getSplat(695 ElementCount::getFixed(XVec->getNumElements()), Zero);696 }697 698 return Builder.CreateSelect(Cond, Zero, One);699}700 701static Value *expandRadiansIntrinsic(CallInst *Orig) {702 Value *X = Orig->getOperand(0);703 Type *Ty = X->getType();704 IRBuilder<> Builder(Orig);705 Value *PiOver180 = ConstantFP::get(Ty, llvm::numbers::pi / 180.0);706 return Builder.CreateFMul(X, PiOver180);707}708 709static bool expandBufferLoadIntrinsic(CallInst *Orig, bool IsRaw) {710 IRBuilder<> Builder(Orig);711 712 Type *BufferTy = Orig->getType()->getStructElementType(0);713 Type *ScalarTy = BufferTy->getScalarType();714 bool IsDouble = ScalarTy->isDoubleTy();715 assert(IsDouble || ScalarTy->isIntegerTy(64) &&716 "Only expand double or int64 scalars or vectors");717 bool IsVector = false;718 unsigned ExtractNum = 2;719 if (auto *VT = dyn_cast<FixedVectorType>(BufferTy)) {720 ExtractNum = 2 * VT->getNumElements();721 IsVector = true;722 assert(IsRaw || ExtractNum == 4 && "TypedBufferLoad vector must be size 2");723 }724 725 SmallVector<Value *, 2> Loads;726 Value *Result = PoisonValue::get(BufferTy);727 unsigned Base = 0;728 // If we need to extract more than 4 i32; we need to break it up into729 // more than one load. LoadNum tells us how many i32s we are loading in730 // each load731 while (ExtractNum > 0) {732 unsigned LoadNum = std::min(ExtractNum, 4u);733 Type *Ty = VectorType::get(Builder.getInt32Ty(), LoadNum, false);734 735 Type *LoadType = StructType::get(Ty, Builder.getInt1Ty());736 Intrinsic::ID LoadIntrinsic = Intrinsic::dx_resource_load_typedbuffer;737 SmallVector<Value *, 3> Args = {Orig->getOperand(0), Orig->getOperand(1)};738 if (IsRaw) {739 LoadIntrinsic = Intrinsic::dx_resource_load_rawbuffer;740 Value *Tmp = Builder.getInt32(4 * Base * 2);741 Args.push_back(Builder.CreateAdd(Orig->getOperand(2), Tmp));742 }743 744 CallInst *Load = Builder.CreateIntrinsic(LoadType, LoadIntrinsic, Args);745 Loads.push_back(Load);746 747 // extract the buffer load's result748 Value *Extract = Builder.CreateExtractValue(Load, {0});749 750 SmallVector<Value *> ExtractElements;751 for (unsigned I = 0; I < LoadNum; ++I)752 ExtractElements.push_back(753 Builder.CreateExtractElement(Extract, Builder.getInt32(I)));754 755 // combine into double(s) or int64(s)756 for (unsigned I = 0; I < LoadNum; I += 2) {757 Value *Combined = nullptr;758 if (IsDouble)759 // For doubles, use dx_asdouble intrinsic760 Combined = Builder.CreateIntrinsic(761 Builder.getDoubleTy(), Intrinsic::dx_asdouble,762 {ExtractElements[I], ExtractElements[I + 1]});763 else {764 // For int64, manually combine two int32s765 // First, zero-extend both values to i64766 Value *Lo =767 Builder.CreateZExt(ExtractElements[I], Builder.getInt64Ty());768 Value *Hi =769 Builder.CreateZExt(ExtractElements[I + 1], Builder.getInt64Ty());770 // Shift the high bits left by 32 bits771 Value *ShiftedHi = Builder.CreateShl(Hi, Builder.getInt64(32));772 // OR the high and low bits together773 Combined = Builder.CreateOr(Lo, ShiftedHi);774 }775 776 if (IsVector)777 Result = Builder.CreateInsertElement(Result, Combined,778 Builder.getInt32((I / 2) + Base));779 else780 Result = Combined;781 }782 783 ExtractNum -= LoadNum;784 Base += LoadNum / 2;785 }786 787 Value *CheckBit = nullptr;788 for (User *U : make_early_inc_range(Orig->users())) {789 // If it's not a ExtractValueInst, we don't know how to790 // handle it791 auto *EVI = dyn_cast<ExtractValueInst>(U);792 if (!EVI)793 llvm_unreachable("Unexpected user of typedbufferload");794 795 ArrayRef<unsigned> Indices = EVI->getIndices();796 assert(Indices.size() == 1);797 798 if (Indices[0] == 0) {799 // Use of the value(s)800 EVI->replaceAllUsesWith(Result);801 } else {802 // Use of the check bit803 assert(Indices[0] == 1 && "Unexpected type for typedbufferload");804 // Note: This does not always match the historical behaviour of DXC.805 // See https://github.com/microsoft/DirectXShaderCompiler/issues/7622806 if (!CheckBit) {807 SmallVector<Value *, 2> CheckBits;808 for (Value *L : Loads)809 CheckBits.push_back(Builder.CreateExtractValue(L, {1}));810 CheckBit = Builder.CreateAnd(CheckBits);811 }812 EVI->replaceAllUsesWith(CheckBit);813 }814 EVI->eraseFromParent();815 }816 Orig->eraseFromParent();817 return true;818}819 820static bool expandBufferStoreIntrinsic(CallInst *Orig, bool IsRaw) {821 IRBuilder<> Builder(Orig);822 823 unsigned ValIndex = IsRaw ? 3 : 2;824 Type *BufferTy = Orig->getFunctionType()->getParamType(ValIndex);825 Type *ScalarTy = BufferTy->getScalarType();826 bool IsDouble = ScalarTy->isDoubleTy();827 assert((IsDouble || ScalarTy->isIntegerTy(64)) &&828 "Only expand double or int64 scalars or vectors");829 830 // Determine if we're dealing with a vector or scalar831 bool IsVector = false;832 unsigned ExtractNum = 2;833 unsigned VecLen = 0;834 if (auto *VT = dyn_cast<FixedVectorType>(BufferTy)) {835 VecLen = VT->getNumElements();836 assert(IsRaw || VecLen == 2 && "TypedBufferStore vector must be size 2");837 ExtractNum = VecLen * 2;838 IsVector = true;839 }840 841 // Create the appropriate vector type for the result842 Type *Int32Ty = Builder.getInt32Ty();843 Type *ResultTy = VectorType::get(Int32Ty, ExtractNum, false);844 Value *Val = PoisonValue::get(ResultTy);845 846 Type *SplitElementTy = Int32Ty;847 if (IsVector)848 SplitElementTy = VectorType::get(SplitElementTy, VecLen, false);849 850 Value *LowBits = nullptr;851 Value *HighBits = nullptr;852 // Split the 64-bit values into 32-bit components853 if (IsDouble) {854 auto *SplitTy = llvm::StructType::get(SplitElementTy, SplitElementTy);855 Value *Split = Builder.CreateIntrinsic(SplitTy, Intrinsic::dx_splitdouble,856 {Orig->getOperand(ValIndex)});857 LowBits = Builder.CreateExtractValue(Split, 0);858 HighBits = Builder.CreateExtractValue(Split, 1);859 } else {860 // Handle int64 type(s)861 Value *InputVal = Orig->getOperand(ValIndex);862 Constant *ShiftAmt = Builder.getInt64(32);863 if (IsVector)864 ShiftAmt =865 ConstantVector::getSplat(ElementCount::getFixed(VecLen), ShiftAmt);866 867 // Split into low and high 32-bit parts868 LowBits = Builder.CreateTrunc(InputVal, SplitElementTy);869 Value *ShiftedVal = Builder.CreateLShr(InputVal, ShiftAmt);870 HighBits = Builder.CreateTrunc(ShiftedVal, SplitElementTy);871 }872 873 if (IsVector) {874 SmallVector<int, 8> Mask;875 for (unsigned I = 0; I < VecLen; ++I) {876 Mask.push_back(I);877 Mask.push_back(I + VecLen);878 }879 Val = Builder.CreateShuffleVector(LowBits, HighBits, Mask);880 } else {881 Val = Builder.CreateInsertElement(Val, LowBits, Builder.getInt32(0));882 Val = Builder.CreateInsertElement(Val, HighBits, Builder.getInt32(1));883 }884 885 // If we need to extract more than 4 i32; we need to break it up into886 // more than one store. StoreNum tells us how many i32s we are storing in887 // each store888 unsigned Base = 0;889 while (ExtractNum > 0) {890 unsigned StoreNum = std::min(ExtractNum, 4u);891 892 Intrinsic::ID StoreIntrinsic = Intrinsic::dx_resource_store_typedbuffer;893 SmallVector<Value *, 4> Args = {Orig->getOperand(0), Orig->getOperand(1)};894 if (IsRaw) {895 StoreIntrinsic = Intrinsic::dx_resource_store_rawbuffer;896 Value *Tmp = Builder.getInt32(4 * Base);897 Args.push_back(Builder.CreateAdd(Orig->getOperand(2), Tmp));898 }899 900 SmallVector<int, 4> Mask;901 for (unsigned I = 0; I < StoreNum; ++I) {902 Mask.push_back(Base + I);903 }904 905 Value *SubVal = Val;906 if (VecLen > 2)907 SubVal = Builder.CreateShuffleVector(Val, Mask);908 909 Args.push_back(SubVal);910 // Create the final intrinsic call911 Builder.CreateIntrinsic(Builder.getVoidTy(), StoreIntrinsic, Args);912 913 ExtractNum -= StoreNum;914 Base += StoreNum;915 }916 Orig->eraseFromParent();917 return true;918}919 920static Intrinsic::ID getMaxForClamp(Intrinsic::ID ClampIntrinsic) {921 if (ClampIntrinsic == Intrinsic::dx_uclamp)922 return Intrinsic::umax;923 if (ClampIntrinsic == Intrinsic::dx_sclamp)924 return Intrinsic::smax;925 assert(ClampIntrinsic == Intrinsic::dx_nclamp);926 return Intrinsic::maxnum;927}928 929static Intrinsic::ID getMinForClamp(Intrinsic::ID ClampIntrinsic) {930 if (ClampIntrinsic == Intrinsic::dx_uclamp)931 return Intrinsic::umin;932 if (ClampIntrinsic == Intrinsic::dx_sclamp)933 return Intrinsic::smin;934 assert(ClampIntrinsic == Intrinsic::dx_nclamp);935 return Intrinsic::minnum;936}937 938static Value *expandClampIntrinsic(CallInst *Orig,939 Intrinsic::ID ClampIntrinsic) {940 Value *X = Orig->getOperand(0);941 Value *Min = Orig->getOperand(1);942 Value *Max = Orig->getOperand(2);943 Type *Ty = X->getType();944 IRBuilder<> Builder(Orig);945 auto *MaxCall = Builder.CreateIntrinsic(Ty, getMaxForClamp(ClampIntrinsic),946 {X, Min}, nullptr, "dx.max");947 return Builder.CreateIntrinsic(Ty, getMinForClamp(ClampIntrinsic),948 {MaxCall, Max}, nullptr, "dx.min");949}950 951static Value *expandDegreesIntrinsic(CallInst *Orig) {952 Value *X = Orig->getOperand(0);953 Type *Ty = X->getType();954 IRBuilder<> Builder(Orig);955 Value *DegreesRatio = ConstantFP::get(Ty, 180.0 * llvm::numbers::inv_pi);956 return Builder.CreateFMul(X, DegreesRatio);957}958 959static Value *expandSignIntrinsic(CallInst *Orig) {960 Value *X = Orig->getOperand(0);961 Type *Ty = X->getType();962 Type *ScalarTy = Ty->getScalarType();963 Type *RetTy = Orig->getType();964 Constant *Zero = Constant::getNullValue(Ty);965 966 IRBuilder<> Builder(Orig);967 968 Value *GT;969 Value *LT;970 if (ScalarTy->isFloatingPointTy()) {971 GT = Builder.CreateFCmpOLT(Zero, X);972 LT = Builder.CreateFCmpOLT(X, Zero);973 } else {974 assert(ScalarTy->isIntegerTy());975 GT = Builder.CreateICmpSLT(Zero, X);976 LT = Builder.CreateICmpSLT(X, Zero);977 }978 979 Value *ZextGT = Builder.CreateZExt(GT, RetTy);980 Value *ZextLT = Builder.CreateZExt(LT, RetTy);981 982 return Builder.CreateSub(ZextGT, ZextLT);983}984 985static bool expandIntrinsic(Function &F, CallInst *Orig) {986 Value *Result = nullptr;987 Intrinsic::ID IntrinsicId = F.getIntrinsicID();988 switch (IntrinsicId) {989 case Intrinsic::abs:990 Result = expandAbs(Orig);991 break;992 case Intrinsic::assume:993 Orig->eraseFromParent();994 return true;995 case Intrinsic::atan2:996 Result = expandAtan2Intrinsic(Orig);997 break;998 case Intrinsic::exp:999 Result = expandExpIntrinsic(Orig);1000 break;1001 case Intrinsic::is_fpclass:1002 Result = expandIsFPClass(Orig);1003 break;1004 case Intrinsic::log:1005 Result = expandLogIntrinsic(Orig);1006 break;1007 case Intrinsic::log10:1008 Result = expandLog10Intrinsic(Orig);1009 break;1010 case Intrinsic::pow:1011 case Intrinsic::powi:1012 Result = expandPowIntrinsic(Orig, IntrinsicId);1013 break;1014 case Intrinsic::dx_all:1015 case Intrinsic::dx_any:1016 Result = expandAnyOrAllIntrinsic(Orig, IntrinsicId);1017 break;1018 case Intrinsic::dx_cross:1019 Result = expandCrossIntrinsic(Orig);1020 break;1021 case Intrinsic::dx_uclamp:1022 case Intrinsic::dx_sclamp:1023 case Intrinsic::dx_nclamp:1024 Result = expandClampIntrinsic(Orig, IntrinsicId);1025 break;1026 case Intrinsic::dx_degrees:1027 Result = expandDegreesIntrinsic(Orig);1028 break;1029 case Intrinsic::dx_isinf:1030 Result = expand16BitIsInf(Orig);1031 break;1032 case Intrinsic::dx_isnan:1033 Result = expand16BitIsNaN(Orig);1034 break;1035 case Intrinsic::dx_lerp:1036 Result = expandLerpIntrinsic(Orig);1037 break;1038 case Intrinsic::dx_normalize:1039 Result = expandNormalizeIntrinsic(Orig);1040 break;1041 case Intrinsic::dx_fdot:1042 Result = expandFloatDotIntrinsic(Orig);1043 break;1044 case Intrinsic::dx_sdot:1045 case Intrinsic::dx_udot:1046 Result = expandIntegerDotIntrinsic(Orig, IntrinsicId);1047 break;1048 case Intrinsic::dx_sign:1049 Result = expandSignIntrinsic(Orig);1050 break;1051 case Intrinsic::dx_step:1052 Result = expandStepIntrinsic(Orig);1053 break;1054 case Intrinsic::dx_radians:1055 Result = expandRadiansIntrinsic(Orig);1056 break;1057 case Intrinsic::dx_resource_load_rawbuffer:1058 if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ true))1059 return true;1060 break;1061 case Intrinsic::dx_resource_store_rawbuffer:1062 if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ true))1063 return true;1064 break;1065 case Intrinsic::dx_resource_load_typedbuffer:1066 if (expandBufferLoadIntrinsic(Orig, /*IsRaw*/ false))1067 return true;1068 break;1069 case Intrinsic::dx_resource_store_typedbuffer:1070 if (expandBufferStoreIntrinsic(Orig, /*IsRaw*/ false))1071 return true;1072 break;1073 case Intrinsic::usub_sat:1074 Result = expandUsubSat(Orig);1075 break;1076 case Intrinsic::vector_reduce_add:1077 case Intrinsic::vector_reduce_fadd:1078 Result = expandVecReduceAdd(Orig, IntrinsicId);1079 break;1080 }1081 if (Result) {1082 Orig->replaceAllUsesWith(Result);1083 Orig->eraseFromParent();1084 return true;1085 }1086 return false;1087}1088 1089static bool expansionIntrinsics(Module &M) {1090 for (auto &F : make_early_inc_range(M.functions())) {1091 if (!isIntrinsicExpansion(F))1092 continue;1093 bool IntrinsicExpanded = false;1094 for (User *U : make_early_inc_range(F.users())) {1095 auto *IntrinsicCall = dyn_cast<CallInst>(U);1096 if (!IntrinsicCall)1097 continue;1098 IntrinsicExpanded = expandIntrinsic(F, IntrinsicCall);1099 }1100 if (F.user_empty() && IntrinsicExpanded)1101 F.eraseFromParent();1102 }1103 return true;1104}1105 1106PreservedAnalyses DXILIntrinsicExpansion::run(Module &M,1107 ModuleAnalysisManager &) {1108 if (expansionIntrinsics(M))1109 return PreservedAnalyses::none();1110 return PreservedAnalyses::all();1111}1112 1113bool DXILIntrinsicExpansionLegacy::runOnModule(Module &M) {1114 return expansionIntrinsics(M);1115}1116 1117char DXILIntrinsicExpansionLegacy::ID = 0;1118 1119INITIALIZE_PASS_BEGIN(DXILIntrinsicExpansionLegacy, DEBUG_TYPE,1120 "DXIL Intrinsic Expansion", false, false)1121INITIALIZE_PASS_END(DXILIntrinsicExpansionLegacy, DEBUG_TYPE,1122 "DXIL Intrinsic Expansion", false, false)1123 1124ModulePass *llvm::createDXILIntrinsicExpansionLegacyPass() {1125 return new DXILIntrinsicExpansionLegacy();1126}1127