1048 lines · cpp
1//===- DXILOpLowering.cpp - Lowering to DXIL operations -------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "DXILOpLowering.h"10#include "DXILConstants.h"11#include "DXILOpBuilder.h"12#include "DXILRootSignature.h"13#include "DXILShaderFlags.h"14#include "DirectX.h"15#include "llvm/ADT/SmallVector.h"16#include "llvm/Analysis/DXILMetadataAnalysis.h"17#include "llvm/Analysis/DXILResource.h"18#include "llvm/CodeGen/Passes.h"19#include "llvm/IR/Constant.h"20#include "llvm/IR/DiagnosticInfo.h"21#include "llvm/IR/IRBuilder.h"22#include "llvm/IR/Instruction.h"23#include "llvm/IR/Instructions.h"24#include "llvm/IR/Intrinsics.h"25#include "llvm/IR/IntrinsicsDirectX.h"26#include "llvm/IR/Module.h"27#include "llvm/IR/PassManager.h"28#include "llvm/IR/Use.h"29#include "llvm/InitializePasses.h"30#include "llvm/Pass.h"31#include "llvm/Support/ErrorHandling.h"32#include "llvm/Support/FormatVariadic.h"33 34#define DEBUG_TYPE "dxil-op-lower"35 36using namespace llvm;37using namespace llvm::dxil;38 39namespace {40class OpLowerer {41 Module &M;42 DXILOpBuilder OpBuilder;43 DXILResourceMap &DRM;44 DXILResourceTypeMap &DRTM;45 const ModuleMetadataInfo &MMDI;46 SmallVector<CallInst *> CleanupCasts;47 Function *CleanupNURI = nullptr;48 49public:50 OpLowerer(Module &M, DXILResourceMap &DRM, DXILResourceTypeMap &DRTM,51 const ModuleMetadataInfo &MMDI)52 : M(M), OpBuilder(M), DRM(DRM), DRTM(DRTM), MMDI(MMDI) {}53 54 /// Replace every call to \c F using \c ReplaceCall, and then erase \c F. If55 /// there is an error replacing a call, we emit a diagnostic and return true.56 [[nodiscard]] bool57 replaceFunction(Function &F,58 llvm::function_ref<Error(CallInst *CI)> ReplaceCall) {59 for (User *U : make_early_inc_range(F.users())) {60 CallInst *CI = dyn_cast<CallInst>(U);61 if (!CI)62 continue;63 64 if (Error E = ReplaceCall(CI)) {65 std::string Message(toString(std::move(E)));66 M.getContext().diagnose(DiagnosticInfoUnsupported(67 *CI->getFunction(), Message, CI->getDebugLoc()));68 69 return true;70 }71 }72 if (F.user_empty())73 F.eraseFromParent();74 return false;75 }76 77 struct IntrinArgSelect {78 enum class Type {79#define DXIL_OP_INTRINSIC_ARG_SELECT_TYPE(name) name,80#include "DXILOperation.inc"81 };82 Type Type;83 int Value;84 };85 86 /// Replaces uses of a struct with uses of an equivalent named struct.87 ///88 /// DXIL operations that return structs give them well known names, so we need89 /// to update uses when we switch from an LLVM intrinsic to an op.90 Error replaceNamedStructUses(CallInst *Intrin, CallInst *DXILOp) {91 auto *IntrinTy = cast<StructType>(Intrin->getType());92 auto *DXILOpTy = cast<StructType>(DXILOp->getType());93 if (!IntrinTy->isLayoutIdentical(DXILOpTy))94 return make_error<StringError>(95 "Type mismatch between intrinsic and DXIL op",96 inconvertibleErrorCode());97 98 for (Use &U : make_early_inc_range(Intrin->uses()))99 if (auto *EVI = dyn_cast<ExtractValueInst>(U.getUser()))100 EVI->setOperand(0, DXILOp);101 else if (auto *IVI = dyn_cast<InsertValueInst>(U.getUser()))102 IVI->setOperand(0, DXILOp);103 else104 return make_error<StringError>("DXIL ops that return structs may only "105 "be used by insert- and extractvalue",106 inconvertibleErrorCode());107 return Error::success();108 }109 110 [[nodiscard]] bool111 replaceFunctionWithOp(Function &F, dxil::OpCode DXILOp,112 ArrayRef<IntrinArgSelect> ArgSelects) {113 return replaceFunction(F, [&](CallInst *CI) -> Error {114 OpBuilder.getIRB().SetInsertPoint(CI);115 SmallVector<Value *> Args;116 if (ArgSelects.size()) {117 for (const IntrinArgSelect &A : ArgSelects) {118 switch (A.Type) {119 case IntrinArgSelect::Type::Index:120 Args.push_back(CI->getArgOperand(A.Value));121 break;122 case IntrinArgSelect::Type::I8:123 Args.push_back(OpBuilder.getIRB().getInt8((uint8_t)A.Value));124 break;125 case IntrinArgSelect::Type::I32:126 Args.push_back(OpBuilder.getIRB().getInt32(A.Value));127 break;128 }129 }130 } else {131 Args.append(CI->arg_begin(), CI->arg_end());132 }133 134 Expected<CallInst *> OpCall =135 OpBuilder.tryCreateOp(DXILOp, Args, CI->getName(), F.getReturnType());136 if (Error E = OpCall.takeError())137 return E;138 139 if (isa<StructType>(CI->getType())) {140 if (Error E = replaceNamedStructUses(CI, *OpCall))141 return E;142 } else143 CI->replaceAllUsesWith(*OpCall);144 145 CI->eraseFromParent();146 return Error::success();147 });148 }149 150 /// Create a cast between a `target("dx")` type and `dx.types.Handle`, which151 /// is intended to be removed by the end of lowering. This is used to allow152 /// lowering of ops which need to change their return or argument types in a153 /// piecemeal way - we can add the casts in to avoid updating all of the uses154 /// or defs, and by the end all of the casts will be redundant.155 Value *createTmpHandleCast(Value *V, Type *Ty) {156 CallInst *Cast = OpBuilder.getIRB().CreateIntrinsic(157 Intrinsic::dx_resource_casthandle, {Ty, V->getType()}, {V});158 CleanupCasts.push_back(Cast);159 return Cast;160 }161 162 void cleanupHandleCasts() {163 SmallVector<CallInst *> ToRemove;164 SmallVector<Function *> CastFns;165 166 for (CallInst *Cast : CleanupCasts) {167 // These casts were only put in to ease the move from `target("dx")` types168 // to `dx.types.Handle in a piecemeal way. At this point, all of the169 // non-cast uses should now be `dx.types.Handle`, and remaining casts170 // should all form pairs to and from the now unused `target("dx")` type.171 CastFns.push_back(Cast->getCalledFunction());172 173 // If the cast is not to `dx.types.Handle`, it should be the first part of174 // the pair. Keep track so we can remove it once it has no more uses.175 if (Cast->getType() != OpBuilder.getHandleType()) {176 ToRemove.push_back(Cast);177 continue;178 }179 // Otherwise, we're the second handle in a pair. Forward the arguments and180 // remove the (second) cast.181 CallInst *Def = cast<CallInst>(Cast->getOperand(0));182 assert(Def->getIntrinsicID() == Intrinsic::dx_resource_casthandle &&183 "Unbalanced pair of temporary handle casts");184 Cast->replaceAllUsesWith(Def->getOperand(0));185 Cast->eraseFromParent();186 }187 for (CallInst *Cast : ToRemove) {188 assert(Cast->user_empty() && "Temporary handle cast still has users");189 Cast->eraseFromParent();190 }191 192 // Deduplicate the cast functions so that we only erase each one once.193 llvm::sort(CastFns);194 CastFns.erase(llvm::unique(CastFns), CastFns.end());195 for (Function *F : CastFns)196 F->eraseFromParent();197 198 CleanupCasts.clear();199 }200 201 void cleanupNonUniformResourceIndexCalls() {202 // Replace all NonUniformResourceIndex calls with their argument.203 if (!CleanupNURI)204 return;205 for (User *U : make_early_inc_range(CleanupNURI->users())) {206 CallInst *CI = dyn_cast<CallInst>(U);207 if (!CI)208 continue;209 CI->replaceAllUsesWith(CI->getArgOperand(0));210 CI->eraseFromParent();211 }212 CleanupNURI->eraseFromParent();213 CleanupNURI = nullptr;214 }215 216 // Remove the resource global associated with the handleFromBinding call217 // instruction and their uses as they aren't needed anymore.218 // TODO: We should verify that all the globals get removed.219 // It's expected we'll need a custom pass in the future that will eliminate220 // the need for this here.221 void removeResourceGlobals(CallInst *CI) {222 for (User *User : make_early_inc_range(CI->users())) {223 if (StoreInst *Store = dyn_cast<StoreInst>(User)) {224 Value *V = Store->getOperand(1);225 Store->eraseFromParent();226 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))227 if (GV->use_empty()) {228 GV->removeDeadConstantUsers();229 GV->eraseFromParent();230 }231 }232 }233 }234 235 void replaceHandleFromBindingCall(CallInst *CI, Value *Replacement) {236 assert(CI->getCalledFunction()->getIntrinsicID() ==237 Intrinsic::dx_resource_handlefrombinding);238 239 removeResourceGlobals(CI);240 241 auto *NameGlobal = dyn_cast<llvm::GlobalVariable>(CI->getArgOperand(4));242 243 CI->replaceAllUsesWith(Replacement);244 CI->eraseFromParent();245 246 if (NameGlobal && NameGlobal->use_empty())247 NameGlobal->removeFromParent();248 }249 250 bool hasNonUniformIndex(Value *IndexOp) {251 if (isa<llvm::Constant>(IndexOp))252 return false;253 254 SmallVector<Value *> WorkList;255 WorkList.push_back(IndexOp);256 257 while (!WorkList.empty()) {258 Value *V = WorkList.pop_back_val();259 if (auto *CI = dyn_cast<CallInst>(V)) {260 if (CI->getCalledFunction()->getIntrinsicID() ==261 Intrinsic::dx_resource_nonuniformindex)262 return true;263 }264 if (auto *U = llvm::dyn_cast<llvm::User>(V)) {265 for (llvm::Value *Op : U->operands()) {266 if (isa<llvm::Constant>(Op))267 continue;268 WorkList.push_back(Op);269 }270 }271 }272 return false;273 }274 275 [[nodiscard]] bool lowerToCreateHandle(Function &F) {276 IRBuilder<> &IRB = OpBuilder.getIRB();277 Type *Int8Ty = IRB.getInt8Ty();278 Type *Int32Ty = IRB.getInt32Ty();279 Type *Int1Ty = IRB.getInt1Ty();280 281 return replaceFunction(F, [&](CallInst *CI) -> Error {282 IRB.SetInsertPoint(CI);283 284 auto *It = DRM.find(CI);285 assert(It != DRM.end() && "Resource not in map?");286 dxil::ResourceInfo &RI = *It;287 288 const auto &Binding = RI.getBinding();289 dxil::ResourceClass RC = DRTM[RI.getHandleTy()].getResourceClass();290 291 Value *IndexOp = CI->getArgOperand(3);292 if (Binding.LowerBound != 0)293 IndexOp = IRB.CreateAdd(IndexOp,294 ConstantInt::get(Int32Ty, Binding.LowerBound));295 296 bool HasNonUniformIndex =297 (Binding.Size == 1) ? false : hasNonUniformIndex(IndexOp);298 std::array<Value *, 4> Args{299 ConstantInt::get(Int8Ty, llvm::to_underlying(RC)),300 ConstantInt::get(Int32Ty, Binding.RecordID), IndexOp,301 ConstantInt::get(Int1Ty, HasNonUniformIndex)};302 Expected<CallInst *> OpCall =303 OpBuilder.tryCreateOp(OpCode::CreateHandle, Args, CI->getName());304 if (Error E = OpCall.takeError())305 return E;306 307 Value *Cast = createTmpHandleCast(*OpCall, CI->getType());308 replaceHandleFromBindingCall(CI, Cast);309 return Error::success();310 });311 }312 313 [[nodiscard]] bool lowerToBindAndAnnotateHandle(Function &F) {314 IRBuilder<> &IRB = OpBuilder.getIRB();315 Type *Int32Ty = IRB.getInt32Ty();316 Type *Int1Ty = IRB.getInt1Ty();317 318 return replaceFunction(F, [&](CallInst *CI) -> Error {319 IRB.SetInsertPoint(CI);320 321 auto *It = DRM.find(CI);322 assert(It != DRM.end() && "Resource not in map?");323 dxil::ResourceInfo &RI = *It;324 325 const auto &Binding = RI.getBinding();326 dxil::ResourceTypeInfo &RTI = DRTM[RI.getHandleTy()];327 dxil::ResourceClass RC = RTI.getResourceClass();328 329 Value *IndexOp = CI->getArgOperand(3);330 if (Binding.LowerBound != 0)331 IndexOp = IRB.CreateAdd(IndexOp,332 ConstantInt::get(Int32Ty, Binding.LowerBound));333 334 std::pair<uint32_t, uint32_t> Props =335 RI.getAnnotateProps(*F.getParent(), RTI);336 337 // For `CreateHandleFromBinding` we need the upper bound rather than the338 // size, so we need to be careful about the difference for "unbounded".339 uint32_t Unbounded = std::numeric_limits<uint32_t>::max();340 uint32_t UpperBound = Binding.Size == Unbounded341 ? Unbounded342 : Binding.LowerBound + Binding.Size - 1;343 Constant *ResBind = OpBuilder.getResBind(Binding.LowerBound, UpperBound,344 Binding.Space, RC);345 bool NonUniformIndex =346 (Binding.Size == 1) ? false : hasNonUniformIndex(IndexOp);347 Constant *NonUniformOp = ConstantInt::get(Int1Ty, NonUniformIndex);348 std::array<Value *, 3> BindArgs{ResBind, IndexOp, NonUniformOp};349 Expected<CallInst *> OpBind = OpBuilder.tryCreateOp(350 OpCode::CreateHandleFromBinding, BindArgs, CI->getName());351 if (Error E = OpBind.takeError())352 return E;353 354 std::array<Value *, 2> AnnotateArgs{355 *OpBind, OpBuilder.getResProps(Props.first, Props.second)};356 Expected<CallInst *> OpAnnotate = OpBuilder.tryCreateOp(357 OpCode::AnnotateHandle, AnnotateArgs,358 CI->hasName() ? CI->getName() + "_annot" : Twine());359 if (Error E = OpAnnotate.takeError())360 return E;361 362 Value *Cast = createTmpHandleCast(*OpAnnotate, CI->getType());363 replaceHandleFromBindingCall(CI, Cast);364 return Error::success();365 });366 }367 368 /// Lower `dx.resource.handlefrombinding` intrinsics depending on the shader369 /// model and taking into account binding information from370 /// DXILResourceAnalysis.371 bool lowerHandleFromBinding(Function &F) {372 if (MMDI.DXILVersion < VersionTuple(1, 6))373 return lowerToCreateHandle(F);374 return lowerToBindAndAnnotateHandle(F);375 }376 377 /// Replace uses of \c Intrin with the values in the `dx.ResRet` of \c Op.378 /// Since we expect to be post-scalarization, make an effort to avoid vectors.379 Error replaceResRetUses(CallInst *Intrin, CallInst *Op, bool HasCheckBit) {380 IRBuilder<> &IRB = OpBuilder.getIRB();381 382 Instruction *OldResult = Intrin;383 Type *OldTy = Intrin->getType();384 385 if (HasCheckBit) {386 auto *ST = cast<StructType>(OldTy);387 388 Value *CheckOp = nullptr;389 Type *Int32Ty = IRB.getInt32Ty();390 for (Use &U : make_early_inc_range(OldResult->uses())) {391 if (auto *EVI = dyn_cast<ExtractValueInst>(U.getUser())) {392 ArrayRef<unsigned> Indices = EVI->getIndices();393 assert(Indices.size() == 1);394 // We're only interested in uses of the check bit for now.395 if (Indices[0] != 1)396 continue;397 if (!CheckOp) {398 Value *NewEVI = IRB.CreateExtractValue(Op, 4);399 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(400 OpCode::CheckAccessFullyMapped, {NewEVI},401 OldResult->hasName() ? OldResult->getName() + "_check"402 : Twine(),403 Int32Ty);404 if (Error E = OpCall.takeError())405 return E;406 CheckOp = *OpCall;407 }408 EVI->replaceAllUsesWith(CheckOp);409 EVI->eraseFromParent();410 }411 }412 413 if (OldResult->use_empty()) {414 // Only the check bit was used, so we're done here.415 OldResult->eraseFromParent();416 return Error::success();417 }418 419 assert(OldResult->hasOneUse() &&420 isa<ExtractValueInst>(*OldResult->user_begin()) &&421 "Expected only use to be extract of first element");422 OldResult = cast<Instruction>(*OldResult->user_begin());423 OldTy = ST->getElementType(0);424 }425 426 // For scalars, we just extract the first element.427 if (!isa<FixedVectorType>(OldTy)) {428 Value *EVI = IRB.CreateExtractValue(Op, 0);429 OldResult->replaceAllUsesWith(EVI);430 OldResult->eraseFromParent();431 if (OldResult != Intrin) {432 assert(Intrin->use_empty() && "Intrinsic still has uses?");433 Intrin->eraseFromParent();434 }435 return Error::success();436 }437 438 std::array<Value *, 4> Extracts = {};439 SmallVector<ExtractElementInst *> DynamicAccesses;440 441 // The users of the operation should all be scalarized, so we attempt to442 // replace the extractelements with extractvalues directly.443 for (Use &U : make_early_inc_range(OldResult->uses())) {444 if (auto *EEI = dyn_cast<ExtractElementInst>(U.getUser())) {445 if (auto *IndexOp = dyn_cast<ConstantInt>(EEI->getIndexOperand())) {446 size_t IndexVal = IndexOp->getZExtValue();447 assert(IndexVal < 4 && "Index into buffer load out of range");448 if (!Extracts[IndexVal])449 Extracts[IndexVal] = IRB.CreateExtractValue(Op, IndexVal);450 EEI->replaceAllUsesWith(Extracts[IndexVal]);451 EEI->eraseFromParent();452 } else {453 DynamicAccesses.push_back(EEI);454 }455 }456 }457 458 const auto *VecTy = cast<FixedVectorType>(OldTy);459 const unsigned N = VecTy->getNumElements();460 461 // If there's a dynamic access we need to round trip through stack memory so462 // that we don't leave vectors around.463 if (!DynamicAccesses.empty()) {464 Type *Int32Ty = IRB.getInt32Ty();465 Constant *Zero = ConstantInt::get(Int32Ty, 0);466 467 Type *ElTy = VecTy->getElementType();468 Type *ArrayTy = ArrayType::get(ElTy, N);469 Value *Alloca = IRB.CreateAlloca(ArrayTy);470 471 for (int I = 0, E = N; I != E; ++I) {472 if (!Extracts[I])473 Extracts[I] = IRB.CreateExtractValue(Op, I);474 Value *GEP = IRB.CreateInBoundsGEP(475 ArrayTy, Alloca, {Zero, ConstantInt::get(Int32Ty, I)});476 IRB.CreateStore(Extracts[I], GEP);477 }478 479 for (ExtractElementInst *EEI : DynamicAccesses) {480 Value *GEP = IRB.CreateInBoundsGEP(ArrayTy, Alloca,481 {Zero, EEI->getIndexOperand()});482 Value *Load = IRB.CreateLoad(ElTy, GEP);483 EEI->replaceAllUsesWith(Load);484 EEI->eraseFromParent();485 }486 }487 488 // If we still have uses, then we're not fully scalarized and need to489 // recreate the vector. This should only happen for things like exported490 // functions from libraries.491 if (!OldResult->use_empty()) {492 for (int I = 0, E = N; I != E; ++I)493 if (!Extracts[I])494 Extracts[I] = IRB.CreateExtractValue(Op, I);495 496 Value *Vec = PoisonValue::get(OldTy);497 for (int I = 0, E = N; I != E; ++I)498 Vec = IRB.CreateInsertElement(Vec, Extracts[I], I);499 OldResult->replaceAllUsesWith(Vec);500 }501 502 OldResult->eraseFromParent();503 if (OldResult != Intrin) {504 assert(Intrin->use_empty() && "Intrinsic still has uses?");505 Intrin->eraseFromParent();506 }507 508 return Error::success();509 }510 511 [[nodiscard]] bool lowerTypedBufferLoad(Function &F, bool HasCheckBit) {512 IRBuilder<> &IRB = OpBuilder.getIRB();513 Type *Int32Ty = IRB.getInt32Ty();514 515 return replaceFunction(F, [&](CallInst *CI) -> Error {516 IRB.SetInsertPoint(CI);517 518 Value *Handle =519 createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType());520 Value *Index0 = CI->getArgOperand(1);521 Value *Index1 = UndefValue::get(Int32Ty);522 523 Type *OldTy = CI->getType();524 if (HasCheckBit)525 OldTy = cast<StructType>(OldTy)->getElementType(0);526 Type *NewRetTy = OpBuilder.getResRetType(OldTy->getScalarType());527 528 std::array<Value *, 3> Args{Handle, Index0, Index1};529 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(530 OpCode::BufferLoad, Args, CI->getName(), NewRetTy);531 if (Error E = OpCall.takeError())532 return E;533 if (Error E = replaceResRetUses(CI, *OpCall, HasCheckBit))534 return E;535 536 return Error::success();537 });538 }539 540 [[nodiscard]] bool lowerRawBufferLoad(Function &F) {541 const DataLayout &DL = F.getDataLayout();542 IRBuilder<> &IRB = OpBuilder.getIRB();543 Type *Int8Ty = IRB.getInt8Ty();544 Type *Int32Ty = IRB.getInt32Ty();545 546 return replaceFunction(F, [&](CallInst *CI) -> Error {547 IRB.SetInsertPoint(CI);548 549 Type *OldTy = cast<StructType>(CI->getType())->getElementType(0);550 Type *ScalarTy = OldTy->getScalarType();551 Type *NewRetTy = OpBuilder.getResRetType(ScalarTy);552 553 Value *Handle =554 createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType());555 Value *Index0 = CI->getArgOperand(1);556 Value *Index1 = CI->getArgOperand(2);557 uint64_t NumElements =558 DL.getTypeSizeInBits(OldTy) / DL.getTypeSizeInBits(ScalarTy);559 Value *Mask = ConstantInt::get(Int8Ty, ~(~0U << NumElements));560 Value *Align =561 ConstantInt::get(Int32Ty, DL.getPrefTypeAlign(ScalarTy).value());562 563 Expected<CallInst *> OpCall =564 MMDI.DXILVersion >= VersionTuple(1, 2)565 ? OpBuilder.tryCreateOp(OpCode::RawBufferLoad,566 {Handle, Index0, Index1, Mask, Align},567 CI->getName(), NewRetTy)568 : OpBuilder.tryCreateOp(OpCode::BufferLoad,569 {Handle, Index0, Index1}, CI->getName(),570 NewRetTy);571 if (Error E = OpCall.takeError())572 return E;573 if (Error E = replaceResRetUses(CI, *OpCall, /*HasCheckBit=*/true))574 return E;575 576 return Error::success();577 });578 }579 580 [[nodiscard]] bool lowerCBufferLoad(Function &F) {581 IRBuilder<> &IRB = OpBuilder.getIRB();582 583 return replaceFunction(F, [&](CallInst *CI) -> Error {584 IRB.SetInsertPoint(CI);585 586 Type *OldTy = cast<StructType>(CI->getType())->getElementType(0);587 Type *ScalarTy = OldTy->getScalarType();588 Type *NewRetTy = OpBuilder.getCBufRetType(ScalarTy);589 590 Value *Handle =591 createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType());592 Value *Index = CI->getArgOperand(1);593 594 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(595 OpCode::CBufferLoadLegacy, {Handle, Index}, CI->getName(), NewRetTy);596 if (Error E = OpCall.takeError())597 return E;598 if (Error E = replaceNamedStructUses(CI, *OpCall))599 return E;600 601 CI->eraseFromParent();602 return Error::success();603 });604 }605 606 [[nodiscard]] bool lowerUpdateCounter(Function &F) {607 IRBuilder<> &IRB = OpBuilder.getIRB();608 Type *Int32Ty = IRB.getInt32Ty();609 610 return replaceFunction(F, [&](CallInst *CI) -> Error {611 IRB.SetInsertPoint(CI);612 Value *Handle =613 createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType());614 Value *Op1 = CI->getArgOperand(1);615 616 std::array<Value *, 2> Args{Handle, Op1};617 618 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(619 OpCode::UpdateCounter, Args, CI->getName(), Int32Ty);620 621 if (Error E = OpCall.takeError())622 return E;623 624 CI->replaceAllUsesWith(*OpCall);625 CI->eraseFromParent();626 return Error::success();627 });628 }629 630 [[nodiscard]] bool lowerGetDimensionsX(Function &F) {631 IRBuilder<> &IRB = OpBuilder.getIRB();632 Type *Int32Ty = IRB.getInt32Ty();633 634 return replaceFunction(F, [&](CallInst *CI) -> Error {635 IRB.SetInsertPoint(CI);636 Value *Handle =637 createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType());638 Value *Undef = UndefValue::get(Int32Ty);639 640 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(641 OpCode::GetDimensions, {Handle, Undef}, CI->getName(), Int32Ty);642 if (Error E = OpCall.takeError())643 return E;644 Value *Dim = IRB.CreateExtractValue(*OpCall, 0);645 646 CI->replaceAllUsesWith(Dim);647 CI->eraseFromParent();648 return Error::success();649 });650 }651 652 [[nodiscard]] bool lowerGetPointer(Function &F) {653 // These should have already been handled in DXILResourceAccess, so we can654 // just clean up the dead prototype.655 assert(F.user_empty() && "getpointer operations should have been removed");656 F.eraseFromParent();657 return false;658 }659 660 [[nodiscard]] bool lowerBufferStore(Function &F, bool IsRaw) {661 const DataLayout &DL = F.getDataLayout();662 IRBuilder<> &IRB = OpBuilder.getIRB();663 Type *Int8Ty = IRB.getInt8Ty();664 Type *Int32Ty = IRB.getInt32Ty();665 666 return replaceFunction(F, [&](CallInst *CI) -> Error {667 IRB.SetInsertPoint(CI);668 669 Value *Handle =670 createTmpHandleCast(CI->getArgOperand(0), OpBuilder.getHandleType());671 Value *Index0 = CI->getArgOperand(1);672 Value *Index1 = IsRaw ? CI->getArgOperand(2) : UndefValue::get(Int32Ty);673 674 Value *Data = CI->getArgOperand(IsRaw ? 3 : 2);675 Type *DataTy = Data->getType();676 Type *ScalarTy = DataTy->getScalarType();677 678 uint64_t NumElements =679 DL.getTypeSizeInBits(DataTy) / DL.getTypeSizeInBits(ScalarTy);680 Value *Mask =681 ConstantInt::get(Int8Ty, IsRaw ? ~(~0U << NumElements) : 15U);682 683 // TODO: check that we only have vector or scalar...684 if (NumElements > 4)685 return make_error<StringError>(686 "Buffer store data must have at most 4 elements",687 inconvertibleErrorCode());688 689 std::array<Value *, 4> DataElements{nullptr, nullptr, nullptr, nullptr};690 if (DataTy == ScalarTy)691 DataElements[0] = Data;692 else {693 // Since we're post-scalarizer, if we see a vector here it's likely694 // constructed solely for the argument of the store. Just use the scalar695 // values from before they're inserted into the temporary.696 auto *IEI = dyn_cast<InsertElementInst>(Data);697 while (IEI) {698 auto *IndexOp = dyn_cast<ConstantInt>(IEI->getOperand(2));699 if (!IndexOp)700 break;701 size_t IndexVal = IndexOp->getZExtValue();702 assert(IndexVal < 4 && "Too many elements for buffer store");703 DataElements[IndexVal] = IEI->getOperand(1);704 IEI = dyn_cast<InsertElementInst>(IEI->getOperand(0));705 }706 }707 708 // If for some reason we weren't able to forward the arguments from the709 // scalarizer artifact, then we may need to actually extract elements from710 // the vector.711 for (int I = 0, E = NumElements; I < E; ++I)712 if (DataElements[I] == nullptr)713 DataElements[I] =714 IRB.CreateExtractElement(Data, ConstantInt::get(Int32Ty, I));715 716 // For any elements beyond the length of the vector, we should fill it up717 // with undef - however, for typed buffers we repeat the first element to718 // match DXC.719 for (int I = NumElements, E = 4; I < E; ++I)720 if (DataElements[I] == nullptr)721 DataElements[I] = IsRaw ? UndefValue::get(ScalarTy) : DataElements[0];722 723 dxil::OpCode Op = OpCode::BufferStore;724 SmallVector<Value *, 9> Args{725 Handle, Index0, Index1, DataElements[0],726 DataElements[1], DataElements[2], DataElements[3], Mask};727 if (IsRaw && MMDI.DXILVersion >= VersionTuple(1, 2)) {728 Op = OpCode::RawBufferStore;729 // RawBufferStore requires the alignment730 Args.push_back(731 ConstantInt::get(Int32Ty, DL.getPrefTypeAlign(ScalarTy).value()));732 }733 Expected<CallInst *> OpCall =734 OpBuilder.tryCreateOp(Op, Args, CI->getName());735 if (Error E = OpCall.takeError())736 return E;737 738 CI->eraseFromParent();739 // Clean up any leftover `insertelement`s740 auto *IEI = dyn_cast<InsertElementInst>(Data);741 while (IEI && IEI->use_empty()) {742 InsertElementInst *Tmp = IEI;743 IEI = dyn_cast<InsertElementInst>(IEI->getOperand(0));744 Tmp->eraseFromParent();745 }746 747 return Error::success();748 });749 }750 751 [[nodiscard]] bool lowerCtpopToCountBits(Function &F) {752 IRBuilder<> &IRB = OpBuilder.getIRB();753 Type *Int32Ty = IRB.getInt32Ty();754 755 return replaceFunction(F, [&](CallInst *CI) -> Error {756 IRB.SetInsertPoint(CI);757 SmallVector<Value *> Args;758 Args.append(CI->arg_begin(), CI->arg_end());759 760 Type *RetTy = Int32Ty;761 Type *FRT = F.getReturnType();762 if (const auto *VT = dyn_cast<VectorType>(FRT))763 RetTy = VectorType::get(RetTy, VT);764 765 Expected<CallInst *> OpCall = OpBuilder.tryCreateOp(766 dxil::OpCode::CountBits, Args, CI->getName(), RetTy);767 if (Error E = OpCall.takeError())768 return E;769 770 // If the result type is 32 bits we can do a direct replacement.771 if (FRT->isIntOrIntVectorTy(32)) {772 CI->replaceAllUsesWith(*OpCall);773 CI->eraseFromParent();774 return Error::success();775 }776 777 unsigned CastOp;778 unsigned CastOp2;779 if (FRT->isIntOrIntVectorTy(16)) {780 CastOp = Instruction::ZExt;781 CastOp2 = Instruction::SExt;782 } else { // must be 64 bits783 assert(FRT->isIntOrIntVectorTy(64) &&784 "Currently only lowering 16, 32, or 64 bit ctpop to CountBits \785 is supported.");786 CastOp = Instruction::Trunc;787 CastOp2 = Instruction::Trunc;788 }789 790 // It is correct to replace the ctpop with the dxil op and791 // remove all casts to i32792 bool NeedsCast = false;793 for (User *User : make_early_inc_range(CI->users())) {794 Instruction *I = dyn_cast<Instruction>(User);795 if (I && (I->getOpcode() == CastOp || I->getOpcode() == CastOp2) &&796 I->getType() == RetTy) {797 I->replaceAllUsesWith(*OpCall);798 I->eraseFromParent();799 } else800 NeedsCast = true;801 }802 803 // It is correct to replace a ctpop with the dxil op and804 // a cast from i32 to the return type of the ctpop805 // the cast is emitted here if there is a non-cast to i32806 // instr which uses the ctpop807 if (NeedsCast) {808 Value *Cast =809 IRB.CreateZExtOrTrunc(*OpCall, F.getReturnType(), "ctpop.cast");810 CI->replaceAllUsesWith(Cast);811 }812 813 CI->eraseFromParent();814 return Error::success();815 });816 }817 818 [[nodiscard]] bool lowerLifetimeIntrinsic(Function &F) {819 IRBuilder<> &IRB = OpBuilder.getIRB();820 return replaceFunction(F, [&](CallInst *CI) -> Error {821 IRB.SetInsertPoint(CI);822 Value *Ptr = CI->getArgOperand(0);823 assert(Ptr->getType()->isPointerTy() &&824 "Expected operand of lifetime intrinsic to be a pointer");825 826 auto ZeroOrUndef = [&](Type *Ty) {827 return MMDI.ValidatorVersion < VersionTuple(1, 6)828 ? Constant::getNullValue(Ty)829 : UndefValue::get(Ty);830 };831 832 Value *Val = nullptr;833 if (auto *GV = dyn_cast<GlobalVariable>(Ptr)) {834 if (GV->hasInitializer() || GV->isExternallyInitialized())835 return Error::success();836 Val = ZeroOrUndef(GV->getValueType());837 } else if (auto *AI = dyn_cast<AllocaInst>(Ptr))838 Val = ZeroOrUndef(AI->getAllocatedType());839 840 assert(Val && "Expected operand of lifetime intrinsic to be a global "841 "variable or alloca instruction");842 IRB.CreateStore(Val, Ptr, false);843 844 CI->eraseFromParent();845 return Error::success();846 });847 }848 849 [[nodiscard]] bool lowerIsFPClass(Function &F) {850 IRBuilder<> &IRB = OpBuilder.getIRB();851 Type *RetTy = IRB.getInt1Ty();852 853 return replaceFunction(F, [&](CallInst *CI) -> Error {854 IRB.SetInsertPoint(CI);855 SmallVector<Value *> Args;856 Value *Fl = CI->getArgOperand(0);857 Args.push_back(Fl);858 859 dxil::OpCode OpCode;860 Value *T = CI->getArgOperand(1);861 auto *TCI = dyn_cast<ConstantInt>(T);862 switch (TCI->getZExtValue()) {863 case FPClassTest::fcInf:864 OpCode = dxil::OpCode::IsInf;865 break;866 case FPClassTest::fcNan:867 OpCode = dxil::OpCode::IsNaN;868 break;869 case FPClassTest::fcNormal:870 OpCode = dxil::OpCode::IsNormal;871 break;872 case FPClassTest::fcFinite:873 OpCode = dxil::OpCode::IsFinite;874 break;875 default:876 SmallString<128> Msg =877 formatv("Unsupported FPClassTest {0} for DXIL Op Lowering",878 TCI->getZExtValue());879 return make_error<StringError>(Msg, inconvertibleErrorCode());880 }881 882 Expected<CallInst *> OpCall =883 OpBuilder.tryCreateOp(OpCode, Args, CI->getName(), RetTy);884 if (Error E = OpCall.takeError())885 return E;886 887 CI->replaceAllUsesWith(*OpCall);888 CI->eraseFromParent();889 return Error::success();890 });891 }892 893 bool lowerIntrinsics() {894 bool Updated = false;895 bool HasErrors = false;896 897 for (Function &F : make_early_inc_range(M.functions())) {898 if (!F.isDeclaration())899 continue;900 Intrinsic::ID ID = F.getIntrinsicID();901 switch (ID) {902 // NOTE: Skip dx_resource_casthandle here. They are903 // resolved after this loop in cleanupHandleCasts.904 case Intrinsic::dx_resource_casthandle:905 // NOTE: llvm.dbg.value is supported as is in DXIL.906 case Intrinsic::dbg_value:907 case Intrinsic::not_intrinsic:908 if (F.use_empty())909 F.eraseFromParent();910 continue;911 default:912 if (F.use_empty())913 F.eraseFromParent();914 else {915 SmallString<128> Msg = formatv(916 "Unsupported intrinsic {0} for DXIL lowering", F.getName());917 M.getContext().emitError(Msg);918 HasErrors |= true;919 }920 break;921 922#define DXIL_OP_INTRINSIC(OpCode, Intrin, ...) \923 case Intrin: \924 HasErrors |= replaceFunctionWithOp( \925 F, OpCode, ArrayRef<IntrinArgSelect>{__VA_ARGS__}); \926 break;927#include "DXILOperation.inc"928 case Intrinsic::dx_resource_handlefrombinding:929 HasErrors |= lowerHandleFromBinding(F);930 break;931 case Intrinsic::dx_resource_getpointer:932 HasErrors |= lowerGetPointer(F);933 break;934 case Intrinsic::dx_resource_nonuniformindex:935 assert(!CleanupNURI &&936 "overloaded llvm.dx.resource.nonuniformindex intrinsics?");937 CleanupNURI = &F;938 break;939 case Intrinsic::dx_resource_load_typedbuffer:940 HasErrors |= lowerTypedBufferLoad(F, /*HasCheckBit=*/true);941 break;942 case Intrinsic::dx_resource_store_typedbuffer:943 HasErrors |= lowerBufferStore(F, /*IsRaw=*/false);944 break;945 case Intrinsic::dx_resource_load_rawbuffer:946 HasErrors |= lowerRawBufferLoad(F);947 break;948 case Intrinsic::dx_resource_store_rawbuffer:949 HasErrors |= lowerBufferStore(F, /*IsRaw=*/true);950 break;951 case Intrinsic::dx_resource_load_cbufferrow_2:952 case Intrinsic::dx_resource_load_cbufferrow_4:953 case Intrinsic::dx_resource_load_cbufferrow_8:954 HasErrors |= lowerCBufferLoad(F);955 break;956 case Intrinsic::dx_resource_updatecounter:957 HasErrors |= lowerUpdateCounter(F);958 break;959 case Intrinsic::dx_resource_getdimensions_x:960 HasErrors |= lowerGetDimensionsX(F);961 break;962 case Intrinsic::ctpop:963 HasErrors |= lowerCtpopToCountBits(F);964 break;965 case Intrinsic::lifetime_start:966 case Intrinsic::lifetime_end:967 if (F.use_empty())968 F.eraseFromParent();969 else {970 if (MMDI.DXILVersion < VersionTuple(1, 6))971 HasErrors |= lowerLifetimeIntrinsic(F);972 else973 continue;974 }975 break;976 case Intrinsic::is_fpclass:977 HasErrors |= lowerIsFPClass(F);978 break;979 }980 Updated = true;981 }982 if (Updated && !HasErrors) {983 cleanupHandleCasts();984 cleanupNonUniformResourceIndexCalls();985 }986 987 return Updated;988 }989};990} // namespace991 992PreservedAnalyses DXILOpLowering::run(Module &M, ModuleAnalysisManager &MAM) {993 DXILResourceMap &DRM = MAM.getResult<DXILResourceAnalysis>(M);994 DXILResourceTypeMap &DRTM = MAM.getResult<DXILResourceTypeAnalysis>(M);995 const ModuleMetadataInfo MMDI = MAM.getResult<DXILMetadataAnalysis>(M);996 997 const bool MadeChanges = OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();998 if (!MadeChanges)999 return PreservedAnalyses::all();1000 PreservedAnalyses PA;1001 PA.preserve<DXILResourceAnalysis>();1002 PA.preserve<DXILMetadataAnalysis>();1003 PA.preserve<ShaderFlagsAnalysis>();1004 PA.preserve<RootSignatureAnalysis>();1005 return PA;1006}1007 1008namespace {1009class DXILOpLoweringLegacy : public ModulePass {1010public:1011 bool runOnModule(Module &M) override {1012 DXILResourceMap &DRM =1013 getAnalysis<DXILResourceWrapperPass>().getResourceMap();1014 DXILResourceTypeMap &DRTM =1015 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();1016 const ModuleMetadataInfo MMDI =1017 getAnalysis<DXILMetadataAnalysisWrapperPass>().getModuleMetadata();1018 1019 return OpLowerer(M, DRM, DRTM, MMDI).lowerIntrinsics();1020 }1021 StringRef getPassName() const override { return "DXIL Op Lowering"; }1022 DXILOpLoweringLegacy() : ModulePass(ID) {}1023 1024 static char ID; // Pass identification.1025 void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {1026 AU.addRequired<DXILResourceTypeWrapperPass>();1027 AU.addRequired<DXILResourceWrapperPass>();1028 AU.addRequired<DXILMetadataAnalysisWrapperPass>();1029 AU.addPreserved<DXILResourceWrapperPass>();1030 AU.addPreserved<DXILMetadataAnalysisWrapperPass>();1031 AU.addPreserved<ShaderFlagsAnalysisWrapper>();1032 AU.addPreserved<RootSignatureAnalysisWrapper>();1033 }1034};1035char DXILOpLoweringLegacy::ID = 0;1036} // end anonymous namespace1037 1038INITIALIZE_PASS_BEGIN(DXILOpLoweringLegacy, DEBUG_TYPE, "DXIL Op Lowering",1039 false, false)1040INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)1041INITIALIZE_PASS_DEPENDENCY(DXILResourceWrapperPass)1042INITIALIZE_PASS_END(DXILOpLoweringLegacy, DEBUG_TYPE, "DXIL Op Lowering", false,1043 false)1044 1045ModulePass *llvm::createDXILOpLoweringLegacyPass() {1046 return new DXILOpLoweringLegacy();1047}1048