467 lines · cpp
1//===-- SPIRVLegalizePointerCast.cpp ----------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// The LLVM IR has multiple legal patterns we cannot lower to Logical SPIR-V.10// This pass modifies such loads to have an IR we can directly lower to valid11// logical SPIR-V.12// OpenCL can avoid this because they rely on ptrcast, which is not supported13// by logical SPIR-V.14//15// This pass relies on the assign_ptr_type intrinsic to deduce the type of the16// pointed values, must replace all occurences of `ptrcast`. This is why17// unhandled cases are reported as unreachable: we MUST cover all cases.18//19// 1. Loading the first element of an array20//21// %array = [10 x i32]22// %value = load i32, ptr %array23//24// LLVM can skip the GEP instruction, and only request loading the first 425// bytes. In logical SPIR-V, we need an OpAccessChain to access the first26// element. This pass will add a getelementptr instruction before the load.27//28//29// 2. Implicit downcast from load30//31// %1 = getelementptr <4 x i32>, ptr %vec4, i64 032// %2 = load <3 x i32>, ptr %133//34// The pointer in the GEP instruction is only used for offset computations,35// but it doesn't NEED to match the pointed type. OpAccessChain however36// requires this. Also, LLVM loads define the bitwidth of the load, not the37// pointer. In this example, we can guess %vec4 is a vec4 thanks to the GEP38// instruction basetype, but we only want to load the first 3 elements, hence39// do a partial load. In logical SPIR-V, this is not legal. What we must do40// is load the full vector (basetype), extract 3 elements, and recombine them41// to form a 3-element vector.42//43//===----------------------------------------------------------------------===//44 45#include "SPIRV.h"46#include "SPIRVSubtarget.h"47#include "SPIRVTargetMachine.h"48#include "SPIRVUtils.h"49#include "llvm/IR/IRBuilder.h"50#include "llvm/IR/IntrinsicInst.h"51#include "llvm/IR/Intrinsics.h"52#include "llvm/IR/IntrinsicsSPIRV.h"53#include "llvm/Transforms/Utils/Cloning.h"54#include "llvm/Transforms/Utils/LowerMemIntrinsics.h"55 56using namespace llvm;57 58namespace {59class SPIRVLegalizePointerCast : public FunctionPass {60 61 // Builds the `spv_assign_type` assigning |Ty| to |Value| at the current62 // builder position.63 void buildAssignType(IRBuilder<> &B, Type *Ty, Value *Arg) {64 Value *OfType = PoisonValue::get(Ty);65 CallInst *AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type,66 {Arg->getType()}, OfType, Arg, {}, B);67 GR->addAssignPtrTypeInstr(Arg, AssignCI);68 }69 70 // Loads parts of the vector of type |SourceType| from the pointer |Source|71 // and create a new vector of type |TargetType|. |TargetType| must be a vector72 // type, and element types of |TargetType| and |SourceType| must match.73 // Returns the loaded value.74 Value *loadVectorFromVector(IRBuilder<> &B, FixedVectorType *SourceType,75 FixedVectorType *TargetType, Value *Source) {76 LoadInst *NewLoad = B.CreateLoad(SourceType, Source);77 buildAssignType(B, SourceType, NewLoad);78 Value *AssignValue = NewLoad;79 if (TargetType->getElementType() != SourceType->getElementType()) {80 const DataLayout &DL = B.GetInsertBlock()->getModule()->getDataLayout();81 [[maybe_unused]] TypeSize TargetTypeSize =82 DL.getTypeSizeInBits(TargetType);83 [[maybe_unused]] TypeSize SourceTypeSize =84 DL.getTypeSizeInBits(SourceType);85 assert(TargetTypeSize == SourceTypeSize);86 AssignValue = B.CreateIntrinsic(Intrinsic::spv_bitcast,87 {TargetType, SourceType}, {NewLoad});88 buildAssignType(B, TargetType, AssignValue);89 return AssignValue;90 }91 92 assert(TargetType->getNumElements() < SourceType->getNumElements());93 SmallVector<int> Mask(/* Size= */ TargetType->getNumElements());94 for (unsigned I = 0; I < TargetType->getNumElements(); ++I)95 Mask[I] = I;96 Value *Output = B.CreateShuffleVector(AssignValue, AssignValue, Mask);97 buildAssignType(B, TargetType, Output);98 return Output;99 }100 101 // Loads the first value in an aggregate pointed by |Source| of containing102 // elements of type |ElementType|. Load flags will be copied from |BadLoad|,103 // which should be the load being legalized. Returns the loaded value.104 Value *loadFirstValueFromAggregate(IRBuilder<> &B, Type *ElementType,105 Value *Source, LoadInst *BadLoad) {106 SmallVector<Type *, 2> Types = {BadLoad->getPointerOperandType(),107 Source->getType()};108 SmallVector<Value *, 8> Args{/* isInBounds= */ B.getInt1(false), Source};109 110 Type *AggregateType = GR->findDeducedElementType(Source);111 assert(AggregateType && "Could not deduce aggregate type");112 buildGEPIndexChain(B, ElementType, AggregateType, Args);113 114 auto *GEP = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});115 GR->buildAssignPtr(B, ElementType, GEP);116 117 LoadInst *LI = B.CreateLoad(ElementType, GEP);118 LI->setAlignment(BadLoad->getAlign());119 buildAssignType(B, ElementType, LI);120 return LI;121 }122 123 // Loads elements from an array and constructs a vector.124 Value *loadVectorFromArray(IRBuilder<> &B, FixedVectorType *TargetType,125 Value *Source) {126 // Load each element of the array.127 SmallVector<Value *, 4> LoadedElements;128 for (unsigned i = 0; i < TargetType->getNumElements(); ++i) {129 // Create a GEP to access the i-th element of the array.130 SmallVector<Type *, 2> Types = {Source->getType(), Source->getType()};131 SmallVector<Value *, 4> Args;132 Args.push_back(B.getInt1(false));133 Args.push_back(Source);134 Args.push_back(B.getInt32(0));135 Args.push_back(ConstantInt::get(B.getInt32Ty(), i));136 auto *ElementPtr = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});137 GR->buildAssignPtr(B, TargetType->getElementType(), ElementPtr);138 139 // Load the value from the element pointer.140 Value *Load = B.CreateLoad(TargetType->getElementType(), ElementPtr);141 buildAssignType(B, TargetType->getElementType(), Load);142 LoadedElements.push_back(Load);143 }144 145 // Build the vector from the loaded elements.146 Value *NewVector = PoisonValue::get(TargetType);147 buildAssignType(B, TargetType, NewVector);148 149 for (unsigned i = 0; i < TargetType->getNumElements(); ++i) {150 Value *Index = B.getInt32(i);151 SmallVector<Type *, 4> Types = {TargetType, TargetType,152 TargetType->getElementType(),153 Index->getType()};154 SmallVector<Value *> Args = {NewVector, LoadedElements[i], Index};155 NewVector = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});156 buildAssignType(B, TargetType, NewVector);157 }158 return NewVector;159 }160 161 // Stores elements from a vector into an array.162 void storeArrayFromVector(IRBuilder<> &B, Value *SrcVector,163 Value *DstArrayPtr, ArrayType *ArrTy,164 Align Alignment) {165 auto *VecTy = cast<FixedVectorType>(SrcVector->getType());166 167 // Ensure the element types of the array and vector are the same.168 assert(VecTy->getElementType() == ArrTy->getElementType() &&169 "Element types of array and vector must be the same.");170 171 for (unsigned i = 0; i < VecTy->getNumElements(); ++i) {172 // Create a GEP to access the i-th element of the array.173 SmallVector<Type *, 2> Types = {DstArrayPtr->getType(),174 DstArrayPtr->getType()};175 SmallVector<Value *, 4> Args;176 Args.push_back(B.getInt1(false));177 Args.push_back(DstArrayPtr);178 Args.push_back(B.getInt32(0));179 Args.push_back(ConstantInt::get(B.getInt32Ty(), i));180 auto *ElementPtr = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});181 GR->buildAssignPtr(B, ArrTy->getElementType(), ElementPtr);182 183 // Extract the element from the vector and store it.184 Value *Index = B.getInt32(i);185 SmallVector<Type *, 3> EltTypes = {VecTy->getElementType(), VecTy,186 Index->getType()};187 SmallVector<Value *, 2> EltArgs = {SrcVector, Index};188 Value *Element =189 B.CreateIntrinsic(Intrinsic::spv_extractelt, {EltTypes}, {EltArgs});190 buildAssignType(B, VecTy->getElementType(), Element);191 192 Types = {Element->getType(), ElementPtr->getType()};193 Args = {Element, ElementPtr, B.getInt16(2), B.getInt8(Alignment.value())};194 B.CreateIntrinsic(Intrinsic::spv_store, {Types}, {Args});195 }196 }197 198 // Replaces the load instruction to get rid of the ptrcast used as source199 // operand.200 void transformLoad(IRBuilder<> &B, LoadInst *LI, Value *CastedOperand,201 Value *OriginalOperand) {202 Type *FromTy = GR->findDeducedElementType(OriginalOperand);203 Type *ToTy = GR->findDeducedElementType(CastedOperand);204 Value *Output = nullptr;205 206 auto *SAT = dyn_cast<ArrayType>(FromTy);207 auto *SVT = dyn_cast<FixedVectorType>(FromTy);208 auto *DVT = dyn_cast<FixedVectorType>(ToTy);209 210 B.SetInsertPoint(LI);211 212 // Destination is the element type of some member of FromTy. For example,213 // loading the 1st element of an array:214 // - float a = array[0];215 if (isTypeFirstElementAggregate(ToTy, FromTy))216 Output = loadFirstValueFromAggregate(B, ToTy, OriginalOperand, LI);217 // Destination is a smaller vector than source or different vector type.218 // - float3 v3 = vector4;219 // - float4 v2 = int4;220 else if (SVT && DVT)221 Output = loadVectorFromVector(B, SVT, DVT, OriginalOperand);222 else if (SAT && DVT && SAT->getElementType() == DVT->getElementType())223 Output = loadVectorFromArray(B, DVT, OriginalOperand);224 else225 llvm_unreachable("Unimplemented implicit down-cast from load.");226 227 GR->replaceAllUsesWith(LI, Output, /* DeleteOld= */ true);228 DeadInstructions.push_back(LI);229 }230 231 // Creates an spv_insertelt instruction (equivalent to llvm's insertelement).232 Value *makeInsertElement(IRBuilder<> &B, Value *Vector, Value *Element,233 unsigned Index) {234 Type *Int32Ty = Type::getInt32Ty(B.getContext());235 SmallVector<Type *, 4> Types = {Vector->getType(), Vector->getType(),236 Element->getType(), Int32Ty};237 SmallVector<Value *> Args = {Vector, Element, B.getInt32(Index)};238 Instruction *NewI =239 B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args});240 buildAssignType(B, Vector->getType(), NewI);241 return NewI;242 }243 244 // Creates an spv_extractelt instruction (equivalent to llvm's245 // extractelement).246 Value *makeExtractElement(IRBuilder<> &B, Type *ElementType, Value *Vector,247 unsigned Index) {248 Type *Int32Ty = Type::getInt32Ty(B.getContext());249 SmallVector<Type *, 3> Types = {ElementType, Vector->getType(), Int32Ty};250 SmallVector<Value *> Args = {Vector, B.getInt32(Index)};251 Instruction *NewI =252 B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args});253 buildAssignType(B, ElementType, NewI);254 return NewI;255 }256 257 // Stores the given Src vector operand into the Dst vector, adjusting the size258 // if required.259 Value *storeVectorFromVector(IRBuilder<> &B, Value *Src, Value *Dst,260 Align Alignment) {261 FixedVectorType *SrcType = cast<FixedVectorType>(Src->getType());262 FixedVectorType *DstType =263 cast<FixedVectorType>(GR->findDeducedElementType(Dst));264 auto dstNumElements = DstType->getNumElements();265 auto srcNumElements = SrcType->getNumElements();266 267 // if the element type differs, it is a bitcast.268 if (DstType->getElementType() != SrcType->getElementType()) {269 // Support bitcast between vectors of different sizes only if270 // the total bitwidth is the same.271 [[maybe_unused]] auto dstBitWidth =272 DstType->getElementType()->getScalarSizeInBits() * dstNumElements;273 [[maybe_unused]] auto srcBitWidth =274 SrcType->getElementType()->getScalarSizeInBits() * srcNumElements;275 assert(dstBitWidth == srcBitWidth &&276 "Unsupported bitcast between vectors of different sizes.");277 278 Src =279 B.CreateIntrinsic(Intrinsic::spv_bitcast, {DstType, SrcType}, {Src});280 buildAssignType(B, DstType, Src);281 SrcType = DstType;282 283 StoreInst *SI = B.CreateStore(Src, Dst);284 SI->setAlignment(Alignment);285 return SI;286 }287 288 assert(DstType->getNumElements() >= SrcType->getNumElements());289 LoadInst *LI = B.CreateLoad(DstType, Dst);290 LI->setAlignment(Alignment);291 Value *OldValues = LI;292 buildAssignType(B, OldValues->getType(), OldValues);293 Value *NewValues = Src;294 295 for (unsigned I = 0; I < SrcType->getNumElements(); ++I) {296 Value *Element =297 makeExtractElement(B, SrcType->getElementType(), NewValues, I);298 OldValues = makeInsertElement(B, OldValues, Element, I);299 }300 301 StoreInst *SI = B.CreateStore(OldValues, Dst);302 SI->setAlignment(Alignment);303 return SI;304 }305 306 void buildGEPIndexChain(IRBuilder<> &B, Type *Search, Type *Aggregate,307 SmallVectorImpl<Value *> &Indices) {308 Indices.push_back(B.getInt32(0));309 310 if (Search == Aggregate)311 return;312 313 if (auto *ST = dyn_cast<StructType>(Aggregate))314 buildGEPIndexChain(B, Search, ST->getTypeAtIndex(0u), Indices);315 else if (auto *AT = dyn_cast<ArrayType>(Aggregate))316 buildGEPIndexChain(B, Search, AT->getElementType(), Indices);317 else if (auto *VT = dyn_cast<FixedVectorType>(Aggregate))318 buildGEPIndexChain(B, Search, VT->getElementType(), Indices);319 else320 llvm_unreachable("Bad access chain?");321 }322 323 // Stores the given Src value into the first entry of the Dst aggregate.324 Value *storeToFirstValueAggregate(IRBuilder<> &B, Value *Src, Value *Dst,325 Type *DstPointeeType, Align Alignment) {326 SmallVector<Type *, 2> Types = {Dst->getType(), Dst->getType()};327 SmallVector<Value *, 8> Args{/* isInBounds= */ B.getInt1(true), Dst};328 buildGEPIndexChain(B, Src->getType(), DstPointeeType, Args);329 auto *GEP = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args});330 GR->buildAssignPtr(B, Src->getType(), GEP);331 StoreInst *SI = B.CreateStore(Src, GEP);332 SI->setAlignment(Alignment);333 return SI;334 }335 336 bool isTypeFirstElementAggregate(Type *Search, Type *Aggregate) {337 if (Search == Aggregate)338 return true;339 if (auto *ST = dyn_cast<StructType>(Aggregate))340 return isTypeFirstElementAggregate(Search, ST->getTypeAtIndex(0u));341 if (auto *VT = dyn_cast<FixedVectorType>(Aggregate))342 return isTypeFirstElementAggregate(Search, VT->getElementType());343 if (auto *AT = dyn_cast<ArrayType>(Aggregate))344 return isTypeFirstElementAggregate(Search, AT->getElementType());345 return false;346 }347 348 // Transforms a store instruction (or SPV intrinsic) using a ptrcast as349 // operand into a valid logical SPIR-V store with no ptrcast.350 void transformStore(IRBuilder<> &B, Instruction *BadStore, Value *Src,351 Value *Dst, Align Alignment) {352 Type *ToTy = GR->findDeducedElementType(Dst);353 Type *FromTy = Src->getType();354 355 auto *S_VT = dyn_cast<FixedVectorType>(FromTy);356 auto *D_ST = dyn_cast<StructType>(ToTy);357 auto *D_VT = dyn_cast<FixedVectorType>(ToTy);358 auto *D_AT = dyn_cast<ArrayType>(ToTy);359 360 B.SetInsertPoint(BadStore);361 if (D_ST && isTypeFirstElementAggregate(FromTy, D_ST))362 storeToFirstValueAggregate(B, Src, Dst, D_ST, Alignment);363 else if (D_VT && S_VT)364 storeVectorFromVector(B, Src, Dst, Alignment);365 else if (D_VT && !S_VT && FromTy == D_VT->getElementType())366 storeToFirstValueAggregate(B, Src, Dst, D_VT, Alignment);367 else if (D_AT && S_VT && S_VT->getElementType() == D_AT->getElementType())368 storeArrayFromVector(B, Src, Dst, D_AT, Alignment);369 else370 llvm_unreachable("Unsupported ptrcast use in store. Please fix.");371 372 DeadInstructions.push_back(BadStore);373 }374 375 void legalizePointerCast(IntrinsicInst *II) {376 Value *CastedOperand = II;377 Value *OriginalOperand = II->getOperand(0);378 379 IRBuilder<> B(II->getContext());380 std::vector<Value *> Users;381 for (Use &U : II->uses())382 Users.push_back(U.getUser());383 384 for (Value *User : Users) {385 if (LoadInst *LI = dyn_cast<LoadInst>(User)) {386 transformLoad(B, LI, CastedOperand, OriginalOperand);387 continue;388 }389 390 if (StoreInst *SI = dyn_cast<StoreInst>(User)) {391 transformStore(B, SI, SI->getValueOperand(), OriginalOperand,392 SI->getAlign());393 continue;394 }395 396 if (IntrinsicInst *Intrin = dyn_cast<IntrinsicInst>(User)) {397 if (Intrin->getIntrinsicID() == Intrinsic::spv_assign_ptr_type) {398 DeadInstructions.push_back(Intrin);399 continue;400 }401 402 if (Intrin->getIntrinsicID() == Intrinsic::spv_gep) {403 GR->replaceAllUsesWith(CastedOperand, OriginalOperand,404 /* DeleteOld= */ false);405 continue;406 }407 408 if (Intrin->getIntrinsicID() == Intrinsic::spv_store) {409 Align Alignment;410 if (ConstantInt *C = dyn_cast<ConstantInt>(Intrin->getOperand(3)))411 Alignment = Align(C->getZExtValue());412 transformStore(B, Intrin, Intrin->getArgOperand(0), OriginalOperand,413 Alignment);414 continue;415 }416 }417 418 llvm_unreachable("Unsupported ptrcast user. Please fix.");419 }420 421 DeadInstructions.push_back(II);422 }423 424public:425 SPIRVLegalizePointerCast(SPIRVTargetMachine *TM) : FunctionPass(ID), TM(TM) {}426 427 bool runOnFunction(Function &F) override {428 const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F);429 GR = ST.getSPIRVGlobalRegistry();430 DeadInstructions.clear();431 432 std::vector<IntrinsicInst *> WorkList;433 for (auto &BB : F) {434 for (auto &I : BB) {435 auto *II = dyn_cast<IntrinsicInst>(&I);436 if (II && II->getIntrinsicID() == Intrinsic::spv_ptrcast)437 WorkList.push_back(II);438 }439 }440 441 for (IntrinsicInst *II : WorkList)442 legalizePointerCast(II);443 444 for (Instruction *I : DeadInstructions)445 I->eraseFromParent();446 447 return DeadInstructions.size() != 0;448 }449 450private:451 SPIRVTargetMachine *TM = nullptr;452 SPIRVGlobalRegistry *GR = nullptr;453 std::vector<Instruction *> DeadInstructions;454 455public:456 static char ID;457};458} // namespace459 460char SPIRVLegalizePointerCast::ID = 0;461INITIALIZE_PASS(SPIRVLegalizePointerCast, "spirv-legalize-bitcast",462 "SPIRV legalize bitcast pass", false, false)463 464FunctionPass *llvm::createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM) {465 return new SPIRVLegalizePointerCast(TM);466}467