565 lines · cpp
1//===- DXILResourceAccess.cpp - Resource access via load/store ------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "DXILResourceAccess.h"10#include "DirectX.h"11#include "llvm/ADT/SetVector.h"12#include "llvm/Analysis/DXILResource.h"13#include "llvm/Frontend/HLSL/HLSLResource.h"14#include "llvm/IR/BasicBlock.h"15#include "llvm/IR/Dominators.h"16#include "llvm/IR/IRBuilder.h"17#include "llvm/IR/Instruction.h"18#include "llvm/IR/Instructions.h"19#include "llvm/IR/IntrinsicInst.h"20#include "llvm/IR/Intrinsics.h"21#include "llvm/IR/IntrinsicsDirectX.h"22#include "llvm/IR/User.h"23#include "llvm/InitializePasses.h"24#include "llvm/Support/FormatVariadic.h"25#include "llvm/Transforms/Utils/ValueMapper.h"26 27#define DEBUG_TYPE "dxil-resource-access"28 29using namespace llvm;30 31static Value *calculateGEPOffset(GetElementPtrInst *GEP, Value *PrevOffset,32 dxil::ResourceTypeInfo &RTI) {33 assert(!PrevOffset && "Non-constant GEP chains not handled yet");34 35 const DataLayout &DL = GEP->getDataLayout();36 37 uint64_t ScalarSize = 1;38 if (RTI.isTyped()) {39 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(0);40 // We need the size of an element in bytes so that we can calculate the41 // offset in elements given a total offset in bytes.42 Type *ScalarType = ContainedType->getScalarType();43 ScalarSize = DL.getTypeSizeInBits(ScalarType) / 8;44 }45 46 APInt ConstantOffset(DL.getIndexTypeSizeInBits(GEP->getType()), 0);47 if (GEP->accumulateConstantOffset(DL, ConstantOffset)) {48 APInt Scaled = ConstantOffset.udiv(ScalarSize);49 return ConstantInt::get(DL.getIndexType(GEP->getType()), Scaled);50 }51 52 unsigned NumIndices = GEP->getNumIndices();53 54 // If we have a single index we're indexing into a top level array. This55 // generally only happens with cbuffers.56 if (NumIndices == 1)57 return *GEP->idx_begin();58 59 // If we have two indices, this should be a simple access through a pointer.60 if (NumIndices == 2) {61 auto IndexIt = GEP->idx_begin();62 assert(cast<ConstantInt>(IndexIt)->getZExtValue() == 0 &&63 "GEP is not indexing through pointer");64 ++IndexIt;65 Value *Offset = *IndexIt;66 assert(++IndexIt == GEP->idx_end() && "Too many indices in GEP");67 return Offset;68 }69 70 llvm_unreachable("Unhandled GEP structure for resource access");71}72 73static void createTypedBufferStore(IntrinsicInst *II, StoreInst *SI,74 Value *Offset, dxil::ResourceTypeInfo &RTI) {75 IRBuilder<> Builder(SI);76 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(0);77 Type *LoadType = StructType::get(ContainedType, Builder.getInt1Ty());78 79 Value *V = SI->getValueOperand();80 if (V->getType() == ContainedType) {81 // V is already the right type.82 assert(!Offset && "store of whole element has offset?");83 } else if (V->getType() == ContainedType->getScalarType()) {84 // We're storing a scalar, so we need to load the current value and only85 // replace the relevant part.86 auto *Load = Builder.CreateIntrinsic(87 LoadType, Intrinsic::dx_resource_load_typedbuffer,88 {II->getOperand(0), II->getOperand(1)});89 auto *Struct = Builder.CreateExtractValue(Load, {0});90 91 // If we have an offset from seeing a GEP earlier, use that. Otherwise, 0.92 if (!Offset)93 Offset = ConstantInt::get(Builder.getInt32Ty(), 0);94 V = Builder.CreateInsertElement(Struct, V, Offset);95 } else {96 llvm_unreachable("Store to typed resource has invalid type");97 }98 99 auto *Inst = Builder.CreateIntrinsic(100 Builder.getVoidTy(), Intrinsic::dx_resource_store_typedbuffer,101 {II->getOperand(0), II->getOperand(1), V});102 SI->replaceAllUsesWith(Inst);103}104 105static void createRawStore(IntrinsicInst *II, StoreInst *SI, Value *Offset) {106 IRBuilder<> Builder(SI);107 108 if (!Offset)109 Offset = ConstantInt::get(Builder.getInt32Ty(), 0);110 Value *V = SI->getValueOperand();111 // TODO: break up larger types112 auto *Inst = Builder.CreateIntrinsic(113 Builder.getVoidTy(), Intrinsic::dx_resource_store_rawbuffer,114 {II->getOperand(0), II->getOperand(1), Offset, V});115 SI->replaceAllUsesWith(Inst);116}117 118static void createStoreIntrinsic(IntrinsicInst *II, StoreInst *SI,119 Value *Offset, dxil::ResourceTypeInfo &RTI) {120 switch (RTI.getResourceKind()) {121 case dxil::ResourceKind::TypedBuffer:122 return createTypedBufferStore(II, SI, Offset, RTI);123 case dxil::ResourceKind::RawBuffer:124 case dxil::ResourceKind::StructuredBuffer:125 return createRawStore(II, SI, Offset);126 case dxil::ResourceKind::Texture1D:127 case dxil::ResourceKind::Texture2D:128 case dxil::ResourceKind::Texture2DMS:129 case dxil::ResourceKind::Texture3D:130 case dxil::ResourceKind::TextureCube:131 case dxil::ResourceKind::Texture1DArray:132 case dxil::ResourceKind::Texture2DArray:133 case dxil::ResourceKind::Texture2DMSArray:134 case dxil::ResourceKind::TextureCubeArray:135 case dxil::ResourceKind::FeedbackTexture2D:136 case dxil::ResourceKind::FeedbackTexture2DArray:137 reportFatalUsageError("DXIL Load not implemented yet");138 return;139 case dxil::ResourceKind::CBuffer:140 case dxil::ResourceKind::Sampler:141 case dxil::ResourceKind::TBuffer:142 case dxil::ResourceKind::RTAccelerationStructure:143 case dxil::ResourceKind::Invalid:144 case dxil::ResourceKind::NumEntries:145 llvm_unreachable("Invalid resource kind for store");146 }147 llvm_unreachable("Unhandled case in switch");148}149 150static void createTypedBufferLoad(IntrinsicInst *II, LoadInst *LI,151 Value *Offset, dxil::ResourceTypeInfo &RTI) {152 IRBuilder<> Builder(LI);153 Type *ContainedType = RTI.getHandleTy()->getTypeParameter(0);154 Type *LoadType = StructType::get(ContainedType, Builder.getInt1Ty());155 156 Value *V =157 Builder.CreateIntrinsic(LoadType, Intrinsic::dx_resource_load_typedbuffer,158 {II->getOperand(0), II->getOperand(1)});159 V = Builder.CreateExtractValue(V, {0});160 161 if (Offset)162 V = Builder.CreateExtractElement(V, Offset);163 164 // If we loaded a <1 x ...> instead of a scalar (presumably to feed a165 // shufflevector), then make sure we're maintaining the resulting type.166 if (auto *VT = dyn_cast<FixedVectorType>(LI->getType()))167 if (VT->getNumElements() == 1 && !isa<FixedVectorType>(V->getType()))168 V = Builder.CreateInsertElement(PoisonValue::get(VT), V,169 Builder.getInt32(0));170 171 LI->replaceAllUsesWith(V);172}173 174static void createRawLoad(IntrinsicInst *II, LoadInst *LI, Value *Offset) {175 IRBuilder<> Builder(LI);176 // TODO: break up larger types177 Type *LoadType = StructType::get(LI->getType(), Builder.getInt1Ty());178 if (!Offset)179 Offset = ConstantInt::get(Builder.getInt32Ty(), 0);180 Value *V =181 Builder.CreateIntrinsic(LoadType, Intrinsic::dx_resource_load_rawbuffer,182 {II->getOperand(0), II->getOperand(1), Offset});183 V = Builder.CreateExtractValue(V, {0});184 185 LI->replaceAllUsesWith(V);186}187 188namespace {189/// Helper for building a `load.cbufferrow` intrinsic given a simple type.190struct CBufferRowIntrin {191 Intrinsic::ID IID;192 Type *RetTy;193 unsigned int EltSize;194 unsigned int NumElts;195 196 CBufferRowIntrin(const DataLayout &DL, Type *Ty) {197 assert(Ty == Ty->getScalarType() && "Expected scalar type");198 199 switch (DL.getTypeSizeInBits(Ty)) {200 case 16:201 IID = Intrinsic::dx_resource_load_cbufferrow_8;202 RetTy = StructType::get(Ty, Ty, Ty, Ty, Ty, Ty, Ty, Ty);203 EltSize = 2;204 NumElts = 8;205 break;206 case 32:207 IID = Intrinsic::dx_resource_load_cbufferrow_4;208 RetTy = StructType::get(Ty, Ty, Ty, Ty);209 EltSize = 4;210 NumElts = 4;211 break;212 case 64:213 IID = Intrinsic::dx_resource_load_cbufferrow_2;214 RetTy = StructType::get(Ty, Ty);215 EltSize = 8;216 NumElts = 2;217 break;218 default:219 llvm_unreachable("Only 16, 32, and 64 bit types supported");220 }221 }222};223} // namespace224 225static void createCBufferLoad(IntrinsicInst *II, LoadInst *LI, Value *Offset,226 dxil::ResourceTypeInfo &RTI) {227 const DataLayout &DL = LI->getDataLayout();228 229 Type *Ty = LI->getType();230 assert(!isa<StructType>(Ty) && "Structs not handled yet");231 CBufferRowIntrin Intrin(DL, Ty->getScalarType());232 233 StringRef Name = LI->getName();234 Value *Handle = II->getOperand(0);235 236 IRBuilder<> Builder(LI);237 238 ConstantInt *GlobalOffset = dyn_cast<ConstantInt>(II->getOperand(1));239 assert(GlobalOffset && "CBuffer getpointer index must be constant");240 241 unsigned int FixedOffset = GlobalOffset->getZExtValue();242 // If we have a further constant offset we can just fold it in to the fixed243 // offset.244 if (auto *ConstOffset = dyn_cast_if_present<ConstantInt>(Offset)) {245 FixedOffset += ConstOffset->getZExtValue();246 Offset = nullptr;247 }248 249 Value *CurrentRow = ConstantInt::get(250 Builder.getInt32Ty(), FixedOffset / hlsl::CBufferRowSizeInBytes);251 unsigned int CurrentIndex =252 (FixedOffset % hlsl::CBufferRowSizeInBytes) / Intrin.EltSize;253 254 assert(!(CurrentIndex && Offset) &&255 "Dynamic indexing into elements of cbuffer rows is not supported");256 // At this point if we have a non-constant offset it has to be an array257 // offset, so we can assume that it's a multiple of the row size.258 if (Offset)259 CurrentRow = FixedOffset ? Builder.CreateAdd(CurrentRow, Offset) : Offset;260 261 auto *CBufLoad = Builder.CreateIntrinsic(262 Intrin.RetTy, Intrin.IID, {Handle, CurrentRow}, nullptr, Name + ".load");263 auto *Elt =264 Builder.CreateExtractValue(CBufLoad, {CurrentIndex++}, Name + ".extract");265 266 // At this point we've loaded the first scalar of our result, but our original267 // type may have been a vector.268 unsigned int Remaining =269 ((DL.getTypeSizeInBits(Ty) / 8) / Intrin.EltSize) - 1;270 if (Remaining == 0) {271 // We only have a single element, so we're done.272 Value *Result = Elt;273 274 // However, if we loaded a <1 x T>, then we need to adjust the type.275 if (auto *VT = dyn_cast<FixedVectorType>(Ty)) {276 assert(VT->getNumElements() == 1 && "Can't have multiple elements here");277 Result = Builder.CreateInsertElement(PoisonValue::get(VT), Result,278 Builder.getInt32(0), Name);279 }280 LI->replaceAllUsesWith(Result);281 return;282 }283 284 // Walk each element and extract it, wrapping to new rows as needed.285 SmallVector<Value *> Extracts{Elt};286 while (Remaining--) {287 CurrentIndex %= Intrin.NumElts;288 289 if (CurrentIndex == 0) {290 CurrentRow = Builder.CreateAdd(CurrentRow,291 ConstantInt::get(Builder.getInt32Ty(), 1));292 CBufLoad = Builder.CreateIntrinsic(Intrin.RetTy, Intrin.IID,293 {Handle, CurrentRow}, nullptr,294 Name + ".load");295 }296 297 Extracts.push_back(Builder.CreateExtractValue(CBufLoad, {CurrentIndex++},298 Name + ".extract"));299 }300 301 // Finally, we build up the original loaded value.302 Value *Result = PoisonValue::get(Ty);303 for (int I = 0, E = Extracts.size(); I < E; ++I)304 Result = Builder.CreateInsertElement(305 Result, Extracts[I], Builder.getInt32(I), Name + formatv(".upto{}", I));306 LI->replaceAllUsesWith(Result);307}308 309static void createLoadIntrinsic(IntrinsicInst *II, LoadInst *LI, Value *Offset,310 dxil::ResourceTypeInfo &RTI) {311 switch (RTI.getResourceKind()) {312 case dxil::ResourceKind::TypedBuffer:313 return createTypedBufferLoad(II, LI, Offset, RTI);314 case dxil::ResourceKind::RawBuffer:315 case dxil::ResourceKind::StructuredBuffer:316 return createRawLoad(II, LI, Offset);317 case dxil::ResourceKind::CBuffer:318 return createCBufferLoad(II, LI, Offset, RTI);319 case dxil::ResourceKind::Texture1D:320 case dxil::ResourceKind::Texture2D:321 case dxil::ResourceKind::Texture2DMS:322 case dxil::ResourceKind::Texture3D:323 case dxil::ResourceKind::TextureCube:324 case dxil::ResourceKind::Texture1DArray:325 case dxil::ResourceKind::Texture2DArray:326 case dxil::ResourceKind::Texture2DMSArray:327 case dxil::ResourceKind::TextureCubeArray:328 case dxil::ResourceKind::FeedbackTexture2D:329 case dxil::ResourceKind::FeedbackTexture2DArray:330 case dxil::ResourceKind::TBuffer:331 reportFatalUsageError("Load not yet implemented for resource type");332 return;333 case dxil::ResourceKind::Sampler:334 case dxil::ResourceKind::RTAccelerationStructure:335 case dxil::ResourceKind::Invalid:336 case dxil::ResourceKind::NumEntries:337 llvm_unreachable("Invalid resource kind for load");338 }339 llvm_unreachable("Unhandled case in switch");340}341 342static SmallVector<Instruction *> collectBlockUseDef(Instruction *Start) {343 SmallPtrSet<Instruction *, 32> Visited;344 SmallVector<Instruction *, 32> Worklist;345 SmallVector<Instruction *> Out;346 auto *BB = Start->getParent();347 348 // Seed with direct users in this block.349 for (User *U : Start->users()) {350 if (auto *I = dyn_cast<Instruction>(U)) {351 if (I->getParent() == BB)352 Worklist.push_back(I);353 }354 }355 356 // BFS over transitive users, constrained to the same block.357 while (!Worklist.empty()) {358 Instruction *I = Worklist.pop_back_val();359 if (!Visited.insert(I).second)360 continue;361 Out.push_back(I);362 363 for (User *U : I->users()) {364 if (auto *J = dyn_cast<Instruction>(U)) {365 if (J->getParent() == BB)366 Worklist.push_back(J);367 }368 }369 for (Use &V : I->operands()) {370 if (auto *J = dyn_cast<Instruction>(V)) {371 if (J->getParent() == BB && V != Start)372 Worklist.push_back(J);373 }374 }375 }376 377 // Order results in program order.378 DenseMap<const Instruction *, unsigned> Ord;379 unsigned Idx = 0;380 for (Instruction &I : *BB)381 Ord[&I] = Idx++;382 383 llvm::sort(Out, [&](Instruction *A, Instruction *B) {384 return Ord.lookup(A) < Ord.lookup(B);385 });386 387 return Out;388}389 390static void phiNodeRemapHelper(PHINode *Phi, BasicBlock *BB,391 IRBuilder<> &Builder,392 SmallVector<Instruction *> &UsesInBlock) {393 394 ValueToValueMapTy VMap;395 Value *Val = Phi->getIncomingValueForBlock(BB);396 VMap[Phi] = Val;397 Builder.SetInsertPoint(&BB->back());398 for (Instruction *I : UsesInBlock) {399 // don't clone over the Phi just remap them400 if (auto *PhiNested = dyn_cast<PHINode>(I)) {401 VMap[PhiNested] = PhiNested->getIncomingValueForBlock(BB);402 continue;403 }404 Instruction *Clone = I->clone();405 RemapInstruction(Clone, VMap,406 RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);407 Builder.Insert(Clone);408 VMap[I] = Clone;409 }410}411 412static void phiNodeReplacement(IntrinsicInst *II,413 SmallVectorImpl<Instruction *> &PrevBBDeadInsts,414 SetVector<BasicBlock *> &DeadBB) {415 SmallVector<Instruction *> CurrBBDeadInsts;416 for (User *U : II->users()) {417 auto *Phi = dyn_cast<PHINode>(U);418 if (!Phi)419 continue;420 421 IRBuilder<> Builder(Phi);422 SmallVector<Instruction *> UsesInBlock = collectBlockUseDef(Phi);423 bool HasReturnUse = isa<ReturnInst>(UsesInBlock.back());424 425 for (unsigned I = 0, E = Phi->getNumIncomingValues(); I < E; I++) {426 auto *CurrIncomingBB = Phi->getIncomingBlock(I);427 phiNodeRemapHelper(Phi, CurrIncomingBB, Builder, UsesInBlock);428 if (HasReturnUse)429 PrevBBDeadInsts.push_back(&CurrIncomingBB->back());430 }431 432 CurrBBDeadInsts.push_back(Phi);433 434 for (Instruction *I : UsesInBlock) {435 CurrBBDeadInsts.push_back(I);436 }437 if (HasReturnUse) {438 BasicBlock *PhiBB = Phi->getParent();439 DeadBB.insert(PhiBB);440 }441 }442 // Traverse the now-dead instructions in RPO and remove them.443 for (Instruction *Dead : llvm::reverse(CurrBBDeadInsts))444 Dead->eraseFromParent();445 CurrBBDeadInsts.clear();446}447 448static void replaceAccess(IntrinsicInst *II, dxil::ResourceTypeInfo &RTI) {449 // Process users keeping track of indexing accumulated from GEPs.450 struct AccessAndOffset {451 User *Access;452 Value *Offset;453 };454 SmallVector<AccessAndOffset> Worklist;455 for (User *U : II->users())456 Worklist.push_back({U, nullptr});457 458 SmallVector<Instruction *> DeadInsts;459 while (!Worklist.empty()) {460 AccessAndOffset Current = Worklist.back();461 Worklist.pop_back();462 463 if (auto *GEP = dyn_cast<GetElementPtrInst>(Current.Access)) {464 IRBuilder<> Builder(GEP);465 466 Value *Offset = calculateGEPOffset(GEP, Current.Offset, RTI);467 for (User *U : GEP->users())468 Worklist.push_back({U, Offset});469 DeadInsts.push_back(GEP);470 471 } else if (auto *SI = dyn_cast<StoreInst>(Current.Access)) {472 assert(SI->getValueOperand() != II && "Pointer escaped!");473 createStoreIntrinsic(II, SI, Current.Offset, RTI);474 DeadInsts.push_back(SI);475 476 } else if (auto *LI = dyn_cast<LoadInst>(Current.Access)) {477 createLoadIntrinsic(II, LI, Current.Offset, RTI);478 DeadInsts.push_back(LI);479 } else480 llvm_unreachable("Unhandled instruction - pointer escaped?");481 }482 483 // Traverse the now-dead instructions in RPO and remove them.484 for (Instruction *Dead : llvm::reverse(DeadInsts))485 Dead->eraseFromParent();486 II->eraseFromParent();487}488 489static bool transformResourcePointers(Function &F, DXILResourceTypeMap &DRTM) {490 SmallVector<std::pair<IntrinsicInst *, dxil::ResourceTypeInfo>> Resources;491 SetVector<BasicBlock *> DeadBB;492 SmallVector<Instruction *> PrevBBDeadInsts;493 for (BasicBlock &BB : make_early_inc_range(F)) {494 for (Instruction &I : make_early_inc_range(BB))495 if (auto *II = dyn_cast<IntrinsicInst>(&I))496 if (II->getIntrinsicID() == Intrinsic::dx_resource_getpointer)497 phiNodeReplacement(II, PrevBBDeadInsts, DeadBB);498 499 for (Instruction &I : BB)500 if (auto *II = dyn_cast<IntrinsicInst>(&I))501 if (II->getIntrinsicID() == Intrinsic::dx_resource_getpointer) {502 auto *HandleTy = cast<TargetExtType>(II->getArgOperand(0)->getType());503 Resources.emplace_back(II, DRTM[HandleTy]);504 }505 }506 for (auto *Dead : PrevBBDeadInsts)507 Dead->eraseFromParent();508 PrevBBDeadInsts.clear();509 for (auto *Dead : DeadBB)510 Dead->eraseFromParent();511 DeadBB.clear();512 513 for (auto &[II, RI] : Resources)514 replaceAccess(II, RI);515 516 return !Resources.empty();517}518 519PreservedAnalyses DXILResourceAccess::run(Function &F,520 FunctionAnalysisManager &FAM) {521 auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F);522 DXILResourceTypeMap *DRTM =523 MAMProxy.getCachedResult<DXILResourceTypeAnalysis>(*F.getParent());524 assert(DRTM && "DXILResourceTypeAnalysis must be available");525 526 bool MadeChanges = transformResourcePointers(F, *DRTM);527 if (!MadeChanges)528 return PreservedAnalyses::all();529 530 PreservedAnalyses PA;531 PA.preserve<DXILResourceTypeAnalysis>();532 PA.preserve<DominatorTreeAnalysis>();533 return PA;534}535 536namespace {537class DXILResourceAccessLegacy : public FunctionPass {538public:539 bool runOnFunction(Function &F) override {540 DXILResourceTypeMap &DRTM =541 getAnalysis<DXILResourceTypeWrapperPass>().getResourceTypeMap();542 return transformResourcePointers(F, DRTM);543 }544 StringRef getPassName() const override { return "DXIL Resource Access"; }545 DXILResourceAccessLegacy() : FunctionPass(ID) {}546 547 static char ID; // Pass identification.548 void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {549 AU.addRequired<DXILResourceTypeWrapperPass>();550 AU.addPreserved<DominatorTreeWrapperPass>();551 }552};553char DXILResourceAccessLegacy::ID = 0;554} // end anonymous namespace555 556INITIALIZE_PASS_BEGIN(DXILResourceAccessLegacy, DEBUG_TYPE,557 "DXIL Resource Access", false, false)558INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)559INITIALIZE_PASS_END(DXILResourceAccessLegacy, DEBUG_TYPE,560 "DXIL Resource Access", false, false)561 562FunctionPass *llvm::createDXILResourceAccessLegacyPass() {563 return new DXILResourceAccessLegacy();564}565