630 lines · cpp
1//===- SPIRVISelLowering.cpp - SPIR-V DAG Lowering Impl ---------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the SPIRVTargetLowering class.10//11//===----------------------------------------------------------------------===//12 13#include "SPIRVISelLowering.h"14#include "SPIRV.h"15#include "SPIRVInstrInfo.h"16#include "SPIRVRegisterBankInfo.h"17#include "SPIRVRegisterInfo.h"18#include "SPIRVSubtarget.h"19#include "llvm/CodeGen/MachineInstrBuilder.h"20#include "llvm/CodeGen/MachineRegisterInfo.h"21#include "llvm/IR/IntrinsicsSPIRV.h"22 23#define DEBUG_TYPE "spirv-lower"24 25using namespace llvm;26 27SPIRVTargetLowering::SPIRVTargetLowering(const TargetMachine &TM,28 const SPIRVSubtarget &ST)29 : TargetLowering(TM, ST), STI(ST) {}30 31// Returns true of the types logically match, as defined in32// https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpCopyLogical.33static bool typesLogicallyMatch(const SPIRVType *Ty1, const SPIRVType *Ty2,34 SPIRVGlobalRegistry &GR) {35 if (Ty1->getOpcode() != Ty2->getOpcode())36 return false;37 38 if (Ty1->getNumOperands() != Ty2->getNumOperands())39 return false;40 41 if (Ty1->getOpcode() == SPIRV::OpTypeArray) {42 // Array must have the same size.43 if (Ty1->getOperand(2).getReg() != Ty2->getOperand(2).getReg())44 return false;45 46 SPIRVType *ElemType1 = GR.getSPIRVTypeForVReg(Ty1->getOperand(1).getReg());47 SPIRVType *ElemType2 = GR.getSPIRVTypeForVReg(Ty2->getOperand(1).getReg());48 return ElemType1 == ElemType2 ||49 typesLogicallyMatch(ElemType1, ElemType2, GR);50 }51 52 if (Ty1->getOpcode() == SPIRV::OpTypeStruct) {53 for (unsigned I = 1; I < Ty1->getNumOperands(); I++) {54 SPIRVType *ElemType1 =55 GR.getSPIRVTypeForVReg(Ty1->getOperand(I).getReg());56 SPIRVType *ElemType2 =57 GR.getSPIRVTypeForVReg(Ty2->getOperand(I).getReg());58 if (ElemType1 != ElemType2 &&59 !typesLogicallyMatch(ElemType1, ElemType2, GR))60 return false;61 }62 return true;63 }64 return false;65}66 67unsigned SPIRVTargetLowering::getNumRegistersForCallingConv(68 LLVMContext &Context, CallingConv::ID CC, EVT VT) const {69 // This code avoids CallLowering fail inside getVectorTypeBreakdown70 // on v3i1 arguments. Maybe we need to return 1 for all types.71 // TODO: remove it once this case is supported by the default implementation.72 if (VT.isVector() && VT.getVectorNumElements() == 3 &&73 (VT.getVectorElementType() == MVT::i1 ||74 VT.getVectorElementType() == MVT::i8))75 return 1;76 if (!VT.isVector() && VT.isInteger() && VT.getSizeInBits() <= 64)77 return 1;78 return getNumRegisters(Context, VT);79}80 81MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,82 CallingConv::ID CC,83 EVT VT) const {84 // This code avoids CallLowering fail inside getVectorTypeBreakdown85 // on v3i1 arguments. Maybe we need to return i32 for all types.86 // TODO: remove it once this case is supported by the default implementation.87 if (VT.isVector() && VT.getVectorNumElements() == 3) {88 if (VT.getVectorElementType() == MVT::i1)89 return MVT::v4i1;90 else if (VT.getVectorElementType() == MVT::i8)91 return MVT::v4i8;92 }93 return getRegisterType(Context, VT);94}95 96bool SPIRVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,97 const CallInst &I,98 MachineFunction &MF,99 unsigned Intrinsic) const {100 unsigned AlignIdx = 3;101 switch (Intrinsic) {102 case Intrinsic::spv_load:103 AlignIdx = 2;104 [[fallthrough]];105 case Intrinsic::spv_store: {106 if (I.getNumOperands() >= AlignIdx + 1) {107 auto *AlignOp = cast<ConstantInt>(I.getOperand(AlignIdx));108 Info.align = Align(AlignOp->getZExtValue());109 }110 Info.flags = static_cast<MachineMemOperand::Flags>(111 cast<ConstantInt>(I.getOperand(AlignIdx - 1))->getZExtValue());112 Info.memVT = MVT::i64;113 // TODO: take into account opaque pointers (don't use getElementType).114 // MVT::getVT(PtrTy->getElementType());115 return true;116 break;117 }118 default:119 break;120 }121 return false;122}123 124std::pair<unsigned, const TargetRegisterClass *>125SPIRVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,126 StringRef Constraint,127 MVT VT) const {128 const TargetRegisterClass *RC = nullptr;129 if (Constraint.starts_with("{"))130 return std::make_pair(0u, RC);131 132 if (VT.isFloatingPoint())133 RC = VT.isVector() ? &SPIRV::vfIDRegClass : &SPIRV::fIDRegClass;134 else if (VT.isInteger())135 RC = VT.isVector() ? &SPIRV::vIDRegClass : &SPIRV::iIDRegClass;136 else137 RC = &SPIRV::iIDRegClass;138 139 return std::make_pair(0u, RC);140}141 142inline Register getTypeReg(MachineRegisterInfo *MRI, Register OpReg) {143 SPIRVType *TypeInst = MRI->getVRegDef(OpReg);144 return TypeInst && TypeInst->getOpcode() == SPIRV::OpFunctionParameter145 ? TypeInst->getOperand(1).getReg()146 : OpReg;147}148 149static void doInsertBitcast(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,150 SPIRVGlobalRegistry &GR, MachineInstr &I,151 Register OpReg, unsigned OpIdx,152 SPIRVType *NewPtrType) {153 MachineIRBuilder MIB(I);154 Register NewReg = createVirtualRegister(NewPtrType, &GR, MRI, MIB.getMF());155 bool Res = MIB.buildInstr(SPIRV::OpBitcast)156 .addDef(NewReg)157 .addUse(GR.getSPIRVTypeID(NewPtrType))158 .addUse(OpReg)159 .constrainAllUses(*STI.getInstrInfo(), *STI.getRegisterInfo(),160 *STI.getRegBankInfo());161 if (!Res)162 report_fatal_error("insert validation bitcast: cannot constrain all uses");163 I.getOperand(OpIdx).setReg(NewReg);164}165 166static SPIRVType *createNewPtrType(SPIRVGlobalRegistry &GR, MachineInstr &I,167 SPIRVType *OpType, bool ReuseType,168 SPIRVType *ResType, const Type *ResTy) {169 SPIRV::StorageClass::StorageClass SC =170 static_cast<SPIRV::StorageClass::StorageClass>(171 OpType->getOperand(1).getImm());172 MachineIRBuilder MIB(I);173 SPIRVType *NewBaseType =174 ReuseType ? ResType175 : GR.getOrCreateSPIRVType(176 ResTy, MIB, SPIRV::AccessQualifier::ReadWrite, false);177 return GR.getOrCreateSPIRVPointerType(NewBaseType, MIB, SC);178}179 180// Insert a bitcast before the instruction to keep SPIR-V code valid181// when there is a type mismatch between results and operand types.182static void validatePtrTypes(const SPIRVSubtarget &STI,183 MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR,184 MachineInstr &I, unsigned OpIdx,185 SPIRVType *ResType, const Type *ResTy = nullptr) {186 // Get operand type187 MachineFunction *MF = I.getParent()->getParent();188 Register OpReg = I.getOperand(OpIdx).getReg();189 Register OpTypeReg = getTypeReg(MRI, OpReg);190 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpTypeReg, MF);191 if (!ResType || !OpType || OpType->getOpcode() != SPIRV::OpTypePointer)192 return;193 // Get operand's pointee type194 Register ElemTypeReg = OpType->getOperand(2).getReg();195 SPIRVType *ElemType = GR.getSPIRVTypeForVReg(ElemTypeReg, MF);196 if (!ElemType)197 return;198 // Check if we need a bitcast to make a statement valid199 bool IsSameMF = MF == ResType->getParent()->getParent();200 bool IsEqualTypes = IsSameMF ? ElemType == ResType201 : GR.getTypeForSPIRVType(ElemType) == ResTy;202 if (IsEqualTypes)203 return;204 // There is a type mismatch between results and operand types205 // and we insert a bitcast before the instruction to keep SPIR-V code valid206 SPIRVType *NewPtrType =207 createNewPtrType(GR, I, OpType, IsSameMF, ResType, ResTy);208 if (!GR.isBitcastCompatible(NewPtrType, OpType))209 report_fatal_error(210 "insert validation bitcast: incompatible result and operand types");211 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);212}213 214// Insert a bitcast before OpGroupWaitEvents if the last argument is a pointer215// that doesn't point to OpTypeEvent.216static void validateGroupWaitEventsPtr(const SPIRVSubtarget &STI,217 MachineRegisterInfo *MRI,218 SPIRVGlobalRegistry &GR,219 MachineInstr &I) {220 constexpr unsigned OpIdx = 2;221 MachineFunction *MF = I.getParent()->getParent();222 Register OpReg = I.getOperand(OpIdx).getReg();223 Register OpTypeReg = getTypeReg(MRI, OpReg);224 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpTypeReg, MF);225 if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)226 return;227 SPIRVType *ElemType = GR.getSPIRVTypeForVReg(OpType->getOperand(2).getReg());228 if (!ElemType || ElemType->getOpcode() == SPIRV::OpTypeEvent)229 return;230 // Insert a bitcast before the instruction to keep SPIR-V code valid.231 LLVMContext &Context = MF->getFunction().getContext();232 SPIRVType *NewPtrType =233 createNewPtrType(GR, I, OpType, false, nullptr,234 TargetExtType::get(Context, "spirv.Event"));235 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);236}237 238static void validateLifetimeStart(const SPIRVSubtarget &STI,239 MachineRegisterInfo *MRI,240 SPIRVGlobalRegistry &GR, MachineInstr &I) {241 Register PtrReg = I.getOperand(0).getReg();242 MachineFunction *MF = I.getParent()->getParent();243 Register PtrTypeReg = getTypeReg(MRI, PtrReg);244 SPIRVType *PtrType = GR.getSPIRVTypeForVReg(PtrTypeReg, MF);245 SPIRVType *PonteeElemType = PtrType ? GR.getPointeeType(PtrType) : nullptr;246 if (!PonteeElemType || PonteeElemType->getOpcode() == SPIRV::OpTypeVoid ||247 (PonteeElemType->getOpcode() == SPIRV::OpTypeInt &&248 PonteeElemType->getOperand(1).getImm() == 8))249 return;250 // To keep the code valid a bitcast must be inserted251 SPIRV::StorageClass::StorageClass SC =252 static_cast<SPIRV::StorageClass::StorageClass>(253 PtrType->getOperand(1).getImm());254 MachineIRBuilder MIB(I);255 LLVMContext &Context = MF->getFunction().getContext();256 SPIRVType *NewPtrType =257 GR.getOrCreateSPIRVPointerType(IntegerType::getInt8Ty(Context), MIB, SC);258 doInsertBitcast(STI, MRI, GR, I, PtrReg, 0, NewPtrType);259}260 261static void validatePtrUnwrapStructField(const SPIRVSubtarget &STI,262 MachineRegisterInfo *MRI,263 SPIRVGlobalRegistry &GR,264 MachineInstr &I, unsigned OpIdx) {265 MachineFunction *MF = I.getParent()->getParent();266 Register OpReg = I.getOperand(OpIdx).getReg();267 Register OpTypeReg = getTypeReg(MRI, OpReg);268 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpTypeReg, MF);269 if (!OpType || OpType->getOpcode() != SPIRV::OpTypePointer)270 return;271 SPIRVType *ElemType = GR.getSPIRVTypeForVReg(OpType->getOperand(2).getReg());272 if (!ElemType || ElemType->getOpcode() != SPIRV::OpTypeStruct ||273 ElemType->getNumOperands() != 2)274 return;275 // It's a structure-wrapper around another type with a single member field.276 SPIRVType *MemberType =277 GR.getSPIRVTypeForVReg(ElemType->getOperand(1).getReg());278 if (!MemberType)279 return;280 unsigned MemberTypeOp = MemberType->getOpcode();281 if (MemberTypeOp != SPIRV::OpTypeVector && MemberTypeOp != SPIRV::OpTypeInt &&282 MemberTypeOp != SPIRV::OpTypeFloat && MemberTypeOp != SPIRV::OpTypeBool)283 return;284 // It's a structure-wrapper around a valid type. Insert a bitcast before the285 // instruction to keep SPIR-V code valid.286 SPIRV::StorageClass::StorageClass SC =287 static_cast<SPIRV::StorageClass::StorageClass>(288 OpType->getOperand(1).getImm());289 MachineIRBuilder MIB(I);290 SPIRVType *NewPtrType = GR.getOrCreateSPIRVPointerType(MemberType, MIB, SC);291 doInsertBitcast(STI, MRI, GR, I, OpReg, OpIdx, NewPtrType);292}293 294// Insert a bitcast before the function call instruction to keep SPIR-V code295// valid when there is a type mismatch between actual and expected types of an296// argument:297// %formal = OpFunctionParameter %formal_type298// ...299// %res = OpFunctionCall %ty %fun %actual ...300// implies that %actual is of %formal_type, and in case of opaque pointers.301// We may need to insert a bitcast to ensure this.302void validateFunCallMachineDef(const SPIRVSubtarget &STI,303 MachineRegisterInfo *DefMRI,304 MachineRegisterInfo *CallMRI,305 SPIRVGlobalRegistry &GR, MachineInstr &FunCall,306 MachineInstr *FunDef) {307 if (FunDef->getOpcode() != SPIRV::OpFunction)308 return;309 unsigned OpIdx = 3;310 for (FunDef = FunDef->getNextNode();311 FunDef && FunDef->getOpcode() == SPIRV::OpFunctionParameter &&312 OpIdx < FunCall.getNumOperands();313 FunDef = FunDef->getNextNode(), OpIdx++) {314 SPIRVType *DefPtrType = DefMRI->getVRegDef(FunDef->getOperand(1).getReg());315 SPIRVType *DefElemType =316 DefPtrType && DefPtrType->getOpcode() == SPIRV::OpTypePointer317 ? GR.getSPIRVTypeForVReg(DefPtrType->getOperand(2).getReg(),318 DefPtrType->getParent()->getParent())319 : nullptr;320 if (DefElemType) {321 const Type *DefElemTy = GR.getTypeForSPIRVType(DefElemType);322 // validatePtrTypes() works in the context if the call site323 // When we process historical records about forward calls324 // we need to switch context to the (forward) call site and325 // then restore it back to the current machine function.326 MachineFunction *CurMF =327 GR.setCurrentFunc(*FunCall.getParent()->getParent());328 validatePtrTypes(STI, CallMRI, GR, FunCall, OpIdx, DefElemType,329 DefElemTy);330 GR.setCurrentFunc(*CurMF);331 }332 }333}334 335// Ensure there is no mismatch between actual and expected arg types: calls336// with a processed definition. Return Function pointer if it's a forward337// call (ahead of definition), and nullptr otherwise.338const Function *validateFunCall(const SPIRVSubtarget &STI,339 MachineRegisterInfo *CallMRI,340 SPIRVGlobalRegistry &GR,341 MachineInstr &FunCall) {342 const GlobalValue *GV = FunCall.getOperand(2).getGlobal();343 const Function *F = dyn_cast<Function>(GV);344 MachineInstr *FunDef =345 const_cast<MachineInstr *>(GR.getFunctionDefinition(F));346 if (!FunDef)347 return F;348 MachineRegisterInfo *DefMRI = &FunDef->getParent()->getParent()->getRegInfo();349 validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, FunCall, FunDef);350 return nullptr;351}352 353// Ensure there is no mismatch between actual and expected arg types: calls354// ahead of a processed definition.355void validateForwardCalls(const SPIRVSubtarget &STI,356 MachineRegisterInfo *DefMRI, SPIRVGlobalRegistry &GR,357 MachineInstr &FunDef) {358 const Function *F = GR.getFunctionByDefinition(&FunDef);359 if (SmallPtrSet<MachineInstr *, 8> *FwdCalls = GR.getForwardCalls(F))360 for (MachineInstr *FunCall : *FwdCalls) {361 MachineRegisterInfo *CallMRI =362 &FunCall->getParent()->getParent()->getRegInfo();363 validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, *FunCall, &FunDef);364 }365}366 367// Validation of an access chain.368void validateAccessChain(const SPIRVSubtarget &STI, MachineRegisterInfo *MRI,369 SPIRVGlobalRegistry &GR, MachineInstr &I) {370 SPIRVType *BaseTypeInst = GR.getSPIRVTypeForVReg(I.getOperand(0).getReg());371 if (BaseTypeInst && BaseTypeInst->getOpcode() == SPIRV::OpTypePointer) {372 SPIRVType *BaseElemType =373 GR.getSPIRVTypeForVReg(BaseTypeInst->getOperand(2).getReg());374 validatePtrTypes(STI, MRI, GR, I, 2, BaseElemType);375 }376}377 378// TODO: the logic of inserting additional bitcast's is to be moved379// to pre-IRTranslation passes eventually380void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const {381 // finalizeLowering() is called twice (see GlobalISel/InstructionSelect.cpp)382 // We'd like to avoid the needless second processing pass.383 if (ProcessedMF.find(&MF) != ProcessedMF.end())384 return;385 386 MachineRegisterInfo *MRI = &MF.getRegInfo();387 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();388 GR.setCurrentFunc(MF);389 for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) {390 MachineBasicBlock *MBB = &*I;391 SmallPtrSet<MachineInstr *, 8> ToMove;392 for (MachineBasicBlock::iterator MBBI = MBB->begin(), MBBE = MBB->end();393 MBBI != MBBE;) {394 MachineInstr &MI = *MBBI++;395 switch (MI.getOpcode()) {396 case SPIRV::OpAtomicLoad:397 case SPIRV::OpAtomicExchange:398 case SPIRV::OpAtomicCompareExchange:399 case SPIRV::OpAtomicCompareExchangeWeak:400 case SPIRV::OpAtomicIIncrement:401 case SPIRV::OpAtomicIDecrement:402 case SPIRV::OpAtomicIAdd:403 case SPIRV::OpAtomicISub:404 case SPIRV::OpAtomicSMin:405 case SPIRV::OpAtomicUMin:406 case SPIRV::OpAtomicSMax:407 case SPIRV::OpAtomicUMax:408 case SPIRV::OpAtomicAnd:409 case SPIRV::OpAtomicOr:410 case SPIRV::OpAtomicXor:411 // for the above listed instructions412 // OpAtomicXXX <ResType>, ptr %Op, ...413 // implies that %Op is a pointer to <ResType>414 case SPIRV::OpLoad:415 // OpLoad <ResType>, ptr %Op implies that %Op is a pointer to <ResType>416 if (enforcePtrTypeCompatibility(MI, 2, 0))417 break;418 419 validatePtrTypes(STI, MRI, GR, MI, 2,420 GR.getSPIRVTypeForVReg(MI.getOperand(0).getReg()));421 break;422 case SPIRV::OpAtomicStore:423 // OpAtomicStore ptr %Op, <Scope>, <Mem>, <Obj>424 // implies that %Op points to the <Obj>'s type425 validatePtrTypes(STI, MRI, GR, MI, 0,426 GR.getSPIRVTypeForVReg(MI.getOperand(3).getReg()));427 break;428 case SPIRV::OpStore:429 // OpStore ptr %Op, <Obj> implies that %Op points to the <Obj>'s type430 validatePtrTypes(STI, MRI, GR, MI, 0,431 GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg()));432 break;433 case SPIRV::OpPtrCastToGeneric:434 case SPIRV::OpGenericCastToPtr:435 case SPIRV::OpGenericCastToPtrExplicit:436 validateAccessChain(STI, MRI, GR, MI);437 break;438 case SPIRV::OpPtrAccessChain:439 case SPIRV::OpInBoundsPtrAccessChain:440 if (MI.getNumOperands() == 4)441 validateAccessChain(STI, MRI, GR, MI);442 break;443 444 case SPIRV::OpFunctionCall:445 // ensure there is no mismatch between actual and expected arg types:446 // calls with a processed definition447 if (MI.getNumOperands() > 3)448 if (const Function *F = validateFunCall(STI, MRI, GR, MI))449 GR.addForwardCall(F, &MI);450 break;451 case SPIRV::OpFunction:452 // ensure there is no mismatch between actual and expected arg types:453 // calls ahead of a processed definition454 validateForwardCalls(STI, MRI, GR, MI);455 break;456 457 // ensure that LLVM IR add/sub instructions result in logical SPIR-V458 // instructions when applied to bool type459 case SPIRV::OpIAddS:460 case SPIRV::OpIAddV:461 case SPIRV::OpISubS:462 case SPIRV::OpISubV:463 if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),464 SPIRV::OpTypeBool))465 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));466 break;467 468 // ensure that LLVM IR bitwise instructions result in logical SPIR-V469 // instructions when applied to bool type470 case SPIRV::OpBitwiseOrS:471 case SPIRV::OpBitwiseOrV:472 if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),473 SPIRV::OpTypeBool))474 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalOr));475 break;476 case SPIRV::OpBitwiseAndS:477 case SPIRV::OpBitwiseAndV:478 if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),479 SPIRV::OpTypeBool))480 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalAnd));481 break;482 case SPIRV::OpBitwiseXorS:483 case SPIRV::OpBitwiseXorV:484 if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(),485 SPIRV::OpTypeBool))486 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual));487 break;488 case SPIRV::OpLifetimeStart:489 case SPIRV::OpLifetimeStop:490 if (MI.getOperand(1).getImm() > 0)491 validateLifetimeStart(STI, MRI, GR, MI);492 break;493 case SPIRV::OpGroupAsyncCopy:494 validatePtrUnwrapStructField(STI, MRI, GR, MI, 3);495 validatePtrUnwrapStructField(STI, MRI, GR, MI, 4);496 break;497 case SPIRV::OpGroupWaitEvents:498 // OpGroupWaitEvents ..., ..., <pointer to OpTypeEvent>499 validateGroupWaitEventsPtr(STI, MRI, GR, MI);500 break;501 case SPIRV::OpConstantI: {502 SPIRVType *Type = GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg());503 if (Type->getOpcode() != SPIRV::OpTypeInt && MI.getOperand(2).isImm() &&504 MI.getOperand(2).getImm() == 0) {505 // Validate the null constant of a target extension type506 MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpConstantNull));507 for (unsigned i = MI.getNumOperands() - 1; i > 1; --i)508 MI.removeOperand(i);509 }510 } break;511 case SPIRV::OpPhi: {512 // Phi refers to a type definition that goes after the Phi513 // instruction, so that the virtual register definition of the type514 // doesn't dominate all uses. Let's place the type definition515 // instruction at the end of the predecessor.516 MachineBasicBlock *Curr = MI.getParent();517 SPIRVType *Type = GR.getSPIRVTypeForVReg(MI.getOperand(1).getReg());518 if (Type->getParent() == Curr && !Curr->pred_empty())519 ToMove.insert(const_cast<MachineInstr *>(Type));520 } break;521 case SPIRV::OpExtInst: {522 // prefetch523 if (!MI.getOperand(2).isImm() || !MI.getOperand(3).isImm() ||524 MI.getOperand(2).getImm() != SPIRV::InstructionSet::OpenCL_std)525 continue;526 switch (MI.getOperand(3).getImm()) {527 case SPIRV::OpenCLExtInst::frexp:528 case SPIRV::OpenCLExtInst::lgamma_r:529 case SPIRV::OpenCLExtInst::remquo: {530 // The last operand must be of a pointer to i32 or vector of i32531 // values.532 MachineIRBuilder MIB(MI);533 SPIRVType *Int32Type = GR.getOrCreateSPIRVIntegerType(32, MIB);534 SPIRVType *RetType = MRI->getVRegDef(MI.getOperand(1).getReg());535 assert(RetType && "Expected return type");536 validatePtrTypes(STI, MRI, GR, MI, MI.getNumOperands() - 1,537 RetType->getOpcode() != SPIRV::OpTypeVector538 ? Int32Type539 : GR.getOrCreateSPIRVVectorType(540 Int32Type, RetType->getOperand(2).getImm(),541 MIB, false));542 } break;543 case SPIRV::OpenCLExtInst::fract:544 case SPIRV::OpenCLExtInst::modf:545 case SPIRV::OpenCLExtInst::sincos:546 // The last operand must be of a pointer to the base type represented547 // by the previous operand.548 assert(MI.getOperand(MI.getNumOperands() - 2).isReg() &&549 "Expected v-reg");550 validatePtrTypes(551 STI, MRI, GR, MI, MI.getNumOperands() - 1,552 GR.getSPIRVTypeForVReg(553 MI.getOperand(MI.getNumOperands() - 2).getReg()));554 break;555 case SPIRV::OpenCLExtInst::prefetch:556 // Expected `ptr` type is a pointer to float, integer or vector, but557 // the pontee value can be wrapped into a struct.558 assert(MI.getOperand(MI.getNumOperands() - 2).isReg() &&559 "Expected v-reg");560 validatePtrUnwrapStructField(STI, MRI, GR, MI,561 MI.getNumOperands() - 2);562 break;563 }564 } break;565 }566 }567 for (MachineInstr *MI : ToMove) {568 MachineBasicBlock *Curr = MI->getParent();569 MachineBasicBlock *Pred = *Curr->pred_begin();570 Pred->insert(Pred->getFirstTerminator(), Curr->remove_instr(MI));571 }572 }573 ProcessedMF.insert(&MF);574 TargetLowering::finalizeLowering(MF);575}576 577// Modifies either operand PtrOpIdx or OpIdx so that the pointee type of578// PtrOpIdx matches the type for operand OpIdx. Returns true if they already579// match or if the instruction was modified to make them match.580bool SPIRVTargetLowering::enforcePtrTypeCompatibility(581 MachineInstr &I, unsigned int PtrOpIdx, unsigned int OpIdx) const {582 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();583 SPIRVType *PtrType = GR.getResultType(I.getOperand(PtrOpIdx).getReg());584 SPIRVType *PointeeType = GR.getPointeeType(PtrType);585 SPIRVType *OpType = GR.getResultType(I.getOperand(OpIdx).getReg());586 587 if (PointeeType == OpType)588 return true;589 590 if (typesLogicallyMatch(PointeeType, OpType, GR)) {591 // Apply OpCopyLogical to OpIdx.592 if (I.getOperand(OpIdx).isDef() &&593 insertLogicalCopyOnResult(I, PointeeType)) {594 return true;595 }596 597 llvm_unreachable("Unable to add OpCopyLogical yet.");598 return false;599 }600 601 return false;602}603 604bool SPIRVTargetLowering::insertLogicalCopyOnResult(605 MachineInstr &I, SPIRVType *NewResultType) const {606 MachineRegisterInfo *MRI = &I.getMF()->getRegInfo();607 SPIRVGlobalRegistry &GR = *STI.getSPIRVGlobalRegistry();608 609 Register NewResultReg =610 createVirtualRegister(NewResultType, &GR, MRI, *I.getMF());611 Register NewTypeReg = GR.getSPIRVTypeID(NewResultType);612 613 assert(llvm::size(I.defs()) == 1 && "Expected only one def");614 MachineOperand &OldResult = *I.defs().begin();615 Register OldResultReg = OldResult.getReg();616 MachineOperand &OldType = *I.uses().begin();617 Register OldTypeReg = OldType.getReg();618 619 OldResult.setReg(NewResultReg);620 OldType.setReg(NewTypeReg);621 622 MachineIRBuilder MIB(*I.getNextNode());623 return MIB.buildInstr(SPIRV::OpCopyLogical)624 .addDef(OldResultReg)625 .addUse(OldTypeReg)626 .addUse(NewResultReg)627 .constrainAllUses(*STI.getInstrInfo(), *STI.getRegisterInfo(),628 *STI.getRegBankInfo());629}630