1111 lines · cpp
1//===-- SPIRVPreLegalizer.cpp - prepare IR for legalization -----*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// The pass prepares IR for legalization: it assigns SPIR-V types to registers10// and removes intrinsics which holded these types during IR translation.11// Also it processes constants and registers them in GR to avoid duplication.12//13//===----------------------------------------------------------------------===//14 15#include "SPIRV.h"16#include "SPIRVSubtarget.h"17#include "SPIRVUtils.h"18#include "llvm/ADT/PostOrderIterator.h"19#include "llvm/CodeGen/GlobalISel/CSEInfo.h"20#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"21#include "llvm/IR/Attributes.h"22#include "llvm/IR/Constants.h"23#include "llvm/IR/IntrinsicsSPIRV.h"24 25#define DEBUG_TYPE "spirv-prelegalizer"26 27using namespace llvm;28 29namespace {30class SPIRVPreLegalizer : public MachineFunctionPass {31public:32 static char ID;33 SPIRVPreLegalizer() : MachineFunctionPass(ID) {}34 bool runOnMachineFunction(MachineFunction &MF) override;35 void getAnalysisUsage(AnalysisUsage &AU) const override;36};37} // namespace38 39void SPIRVPreLegalizer::getAnalysisUsage(AnalysisUsage &AU) const {40 AU.addPreserved<GISelValueTrackingAnalysisLegacy>();41 MachineFunctionPass::getAnalysisUsage(AU);42}43 44static void45addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,46 const SPIRVSubtarget &STI,47 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {48 MachineRegisterInfo &MRI = MF.getRegInfo();49 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;50 SmallVector<MachineInstr *, 10> ToErase, ToEraseComposites;51 for (MachineBasicBlock &MBB : MF) {52 for (MachineInstr &MI : MBB) {53 if (!isSpvIntrinsic(MI, Intrinsic::spv_track_constant))54 continue;55 ToErase.push_back(&MI);56 Register SrcReg = MI.getOperand(2).getReg();57 auto *Const =58 cast<Constant>(cast<ConstantAsMetadata>(59 MI.getOperand(3).getMetadata()->getOperand(0))60 ->getValue());61 if (auto *GV = dyn_cast<GlobalValue>(Const)) {62 Register Reg = GR->find(GV, &MF);63 if (!Reg.isValid()) {64 GR->add(GV, MRI.getVRegDef(SrcReg));65 GR->addGlobalObject(GV, &MF, SrcReg);66 } else67 RegsAlreadyAddedToDT[&MI] = Reg;68 } else {69 Register Reg = GR->find(Const, &MF);70 if (!Reg.isValid()) {71 if (auto *ConstVec = dyn_cast<ConstantDataVector>(Const)) {72 auto *BuildVec = MRI.getVRegDef(SrcReg);73 assert(BuildVec &&74 BuildVec->getOpcode() == TargetOpcode::G_BUILD_VECTOR);75 GR->add(Const, BuildVec);76 for (unsigned i = 0; i < ConstVec->getNumElements(); ++i) {77 // Ensure that OpConstantComposite reuses a constant when it's78 // already created and available in the same machine function.79 Constant *ElemConst = ConstVec->getElementAsConstant(i);80 Register ElemReg = GR->find(ElemConst, &MF);81 if (!ElemReg.isValid())82 GR->add(ElemConst,83 MRI.getVRegDef(BuildVec->getOperand(1 + i).getReg()));84 else85 BuildVec->getOperand(1 + i).setReg(ElemReg);86 }87 }88 if (Const->getType()->isTargetExtTy()) {89 // remember association so that we can restore it when assign types90 MachineInstr *SrcMI = MRI.getVRegDef(SrcReg);91 if (SrcMI)92 GR->add(Const, SrcMI);93 if (SrcMI && (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT ||94 SrcMI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF))95 TargetExtConstTypes[SrcMI] = Const->getType();96 if (Const->isNullValue()) {97 MachineBasicBlock &DepMBB = MF.front();98 MachineIRBuilder MIB(DepMBB, DepMBB.getFirstNonPHI());99 SPIRVType *ExtType = GR->getOrCreateSPIRVType(100 Const->getType(), MIB, SPIRV::AccessQualifier::ReadWrite,101 true);102 assert(SrcMI && "Expected source instruction to be valid");103 SrcMI->setDesc(STI.getInstrInfo()->get(SPIRV::OpConstantNull));104 SrcMI->addOperand(MachineOperand::CreateReg(105 GR->getSPIRVTypeID(ExtType), false));106 }107 }108 } else {109 RegsAlreadyAddedToDT[&MI] = Reg;110 // This MI is unused and will be removed. If the MI uses111 // const_composite, it will be unused and should be removed too.112 assert(MI.getOperand(2).isReg() && "Reg operand is expected");113 MachineInstr *SrcMI = MRI.getVRegDef(MI.getOperand(2).getReg());114 if (SrcMI && isSpvIntrinsic(*SrcMI, Intrinsic::spv_const_composite))115 ToEraseComposites.push_back(SrcMI);116 }117 }118 }119 }120 for (MachineInstr *MI : ToErase) {121 Register Reg = MI->getOperand(2).getReg();122 auto It = RegsAlreadyAddedToDT.find(MI);123 if (It != RegsAlreadyAddedToDT.end())124 Reg = It->second;125 auto *RC = MRI.getRegClassOrNull(MI->getOperand(0).getReg());126 if (!MRI.getRegClassOrNull(Reg) && RC)127 MRI.setRegClass(Reg, RC);128 MRI.replaceRegWith(MI->getOperand(0).getReg(), Reg);129 GR->invalidateMachineInstr(MI);130 MI->eraseFromParent();131 }132 for (MachineInstr *MI : ToEraseComposites) {133 GR->invalidateMachineInstr(MI);134 MI->eraseFromParent();135 }136}137 138static void foldConstantsIntoIntrinsics(MachineFunction &MF,139 SPIRVGlobalRegistry *GR,140 MachineIRBuilder MIB) {141 SmallVector<MachineInstr *, 64> ToErase;142 for (MachineBasicBlock &MBB : MF) {143 for (MachineInstr &MI : MBB) {144 if (!isSpvIntrinsic(MI, Intrinsic::spv_assign_name))145 continue;146 const MDNode *MD = MI.getOperand(2).getMetadata();147 StringRef ValueName = cast<MDString>(MD->getOperand(0))->getString();148 if (ValueName.size() > 0) {149 MIB.setInsertPt(*MI.getParent(), MI);150 buildOpName(MI.getOperand(1).getReg(), ValueName, MIB);151 }152 ToErase.push_back(&MI);153 }154 for (MachineInstr *MI : ToErase) {155 GR->invalidateMachineInstr(MI);156 MI->eraseFromParent();157 }158 ToErase.clear();159 }160}161 162static MachineInstr *findAssignTypeInstr(Register Reg,163 MachineRegisterInfo *MRI) {164 for (MachineRegisterInfo::use_instr_iterator I = MRI->use_instr_begin(Reg),165 IE = MRI->use_instr_end();166 I != IE; ++I) {167 MachineInstr *UseMI = &*I;168 if ((isSpvIntrinsic(*UseMI, Intrinsic::spv_assign_ptr_type) ||169 isSpvIntrinsic(*UseMI, Intrinsic::spv_assign_type)) &&170 UseMI->getOperand(1).getReg() == Reg)171 return UseMI;172 }173 return nullptr;174}175 176static void buildOpBitcast(SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,177 Register ResVReg, Register OpReg) {178 SPIRVType *ResType = GR->getSPIRVTypeForVReg(ResVReg);179 SPIRVType *OpType = GR->getSPIRVTypeForVReg(OpReg);180 assert(ResType && OpType && "Operand types are expected");181 if (!GR->isBitcastCompatible(ResType, OpType))182 report_fatal_error("incompatible result and operand types in a bitcast");183 MachineRegisterInfo *MRI = MIB.getMRI();184 if (!MRI->getRegClassOrNull(ResVReg))185 MRI->setRegClass(ResVReg, GR->getRegClass(ResType));186 if (ResType == OpType)187 MIB.buildInstr(TargetOpcode::COPY).addDef(ResVReg).addUse(OpReg);188 else189 MIB.buildInstr(SPIRV::OpBitcast)190 .addDef(ResVReg)191 .addUse(GR->getSPIRVTypeID(ResType))192 .addUse(OpReg);193}194 195// We lower G_BITCAST to OpBitcast here to avoid a MachineVerifier error.196// The verifier checks if the source and destination LLTs of a G_BITCAST are197// different, but this check is too strict for SPIR-V's typed pointers, which198// may have the same LLT but different SPIRVType (e.g. pointers to different199// pointee types). By lowering to OpBitcast here, we bypass the verifier's200// check. See discussion in https://github.com/llvm/llvm-project/pull/110270201// for more context.202//203// We also handle the llvm.spv.bitcast intrinsic here. If the source and204// destination SPIR-V types are the same, we lower it to a COPY to enable205// further optimizations like copy propagation.206static void lowerBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,207 MachineIRBuilder MIB) {208 SmallVector<MachineInstr *, 16> ToErase;209 for (MachineBasicBlock &MBB : MF) {210 for (MachineInstr &MI : MBB) {211 if (isSpvIntrinsic(MI, Intrinsic::spv_bitcast)) {212 Register DstReg = MI.getOperand(0).getReg();213 Register SrcReg = MI.getOperand(2).getReg();214 SPIRVType *DstType = GR->getSPIRVTypeForVReg(DstReg);215 assert(216 DstType &&217 "Expected destination SPIR-V type to have been assigned already.");218 SPIRVType *SrcType = GR->getSPIRVTypeForVReg(SrcReg);219 assert(SrcType &&220 "Expected source SPIR-V type to have been assigned already.");221 if (DstType == SrcType) {222 MIB.setInsertPt(*MI.getParent(), MI);223 MIB.buildCopy(DstReg, SrcReg);224 ToErase.push_back(&MI);225 continue;226 }227 }228 229 if (MI.getOpcode() != TargetOpcode::G_BITCAST)230 continue;231 232 MIB.setInsertPt(*MI.getParent(), MI);233 buildOpBitcast(GR, MIB, MI.getOperand(0).getReg(),234 MI.getOperand(1).getReg());235 ToErase.push_back(&MI);236 }237 }238 for (MachineInstr *MI : ToErase) {239 GR->invalidateMachineInstr(MI);240 MI->eraseFromParent();241 }242}243 244static void insertBitcasts(MachineFunction &MF, SPIRVGlobalRegistry *GR,245 MachineIRBuilder MIB) {246 // Get access to information about available extensions247 const SPIRVSubtarget *ST =248 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());249 SmallVector<MachineInstr *, 10> ToErase;250 for (MachineBasicBlock &MBB : MF) {251 for (MachineInstr &MI : MBB) {252 if (!isSpvIntrinsic(MI, Intrinsic::spv_ptrcast))253 continue;254 assert(MI.getOperand(2).isReg());255 MIB.setInsertPt(*MI.getParent(), MI);256 ToErase.push_back(&MI);257 Register Def = MI.getOperand(0).getReg();258 Register Source = MI.getOperand(2).getReg();259 Type *ElemTy = getMDOperandAsType(MI.getOperand(3).getMetadata(), 0);260 SPIRVType *AssignedPtrType = GR->getOrCreateSPIRVPointerType(261 ElemTy, MI,262 addressSpaceToStorageClass(MI.getOperand(4).getImm(), *ST));263 264 // If the ptrcast would be redundant, replace all uses with the source265 // register.266 MachineRegisterInfo *MRI = MIB.getMRI();267 if (GR->getSPIRVTypeForVReg(Source) == AssignedPtrType) {268 // Erase Def's assign type instruction if we are going to replace Def.269 if (MachineInstr *AssignMI = findAssignTypeInstr(Def, MRI))270 ToErase.push_back(AssignMI);271 MRI->replaceRegWith(Def, Source);272 } else {273 if (!GR->getSPIRVTypeForVReg(Def, &MF))274 GR->assignSPIRVTypeToVReg(AssignedPtrType, Def, MF);275 MIB.buildBitcast(Def, Source);276 }277 }278 }279 for (MachineInstr *MI : ToErase) {280 GR->invalidateMachineInstr(MI);281 MI->eraseFromParent();282 }283}284 285// Translating GV, IRTranslator sometimes generates following IR:286// %1 = G_GLOBAL_VALUE287// %2 = COPY %1288// %3 = G_ADDRSPACE_CAST %2289//290// or291//292// %1 = G_ZEXT %2293// G_MEMCPY ... %2 ...294//295// New registers have no SPIRVType and no register class info.296//297// Set SPIRVType for GV, propagate it from GV to other instructions,298// also set register classes.299static SPIRVType *propagateSPIRVType(MachineInstr *MI, SPIRVGlobalRegistry *GR,300 MachineRegisterInfo &MRI,301 MachineIRBuilder &MIB) {302 SPIRVType *SpvType = nullptr;303 assert(MI && "Machine instr is expected");304 if (MI->getOperand(0).isReg()) {305 Register Reg = MI->getOperand(0).getReg();306 SpvType = GR->getSPIRVTypeForVReg(Reg);307 if (!SpvType) {308 switch (MI->getOpcode()) {309 case TargetOpcode::G_FCONSTANT:310 case TargetOpcode::G_CONSTANT: {311 MIB.setInsertPt(*MI->getParent(), MI);312 Type *Ty = MI->getOperand(1).getCImm()->getType();313 SpvType = GR->getOrCreateSPIRVType(314 Ty, MIB, SPIRV::AccessQualifier::ReadWrite, true);315 break;316 }317 case TargetOpcode::G_GLOBAL_VALUE: {318 MIB.setInsertPt(*MI->getParent(), MI);319 const GlobalValue *Global = MI->getOperand(1).getGlobal();320 Type *ElementTy = toTypedPointer(GR->getDeducedGlobalValueType(Global));321 auto *Ty = TypedPointerType::get(ElementTy,322 Global->getType()->getAddressSpace());323 SpvType = GR->getOrCreateSPIRVType(324 Ty, MIB, SPIRV::AccessQualifier::ReadWrite, true);325 break;326 }327 case TargetOpcode::G_ANYEXT:328 case TargetOpcode::G_SEXT:329 case TargetOpcode::G_ZEXT: {330 if (MI->getOperand(1).isReg()) {331 if (MachineInstr *DefInstr =332 MRI.getVRegDef(MI->getOperand(1).getReg())) {333 if (SPIRVType *Def = propagateSPIRVType(DefInstr, GR, MRI, MIB)) {334 unsigned CurrentBW = GR->getScalarOrVectorBitWidth(Def);335 unsigned ExpectedBW =336 std::max(MRI.getType(Reg).getScalarSizeInBits(), CurrentBW);337 unsigned NumElements = GR->getScalarOrVectorComponentCount(Def);338 SpvType = GR->getOrCreateSPIRVIntegerType(ExpectedBW, MIB);339 if (NumElements > 1)340 SpvType = GR->getOrCreateSPIRVVectorType(SpvType, NumElements,341 MIB, true);342 }343 }344 }345 break;346 }347 case TargetOpcode::G_PTRTOINT:348 SpvType = GR->getOrCreateSPIRVIntegerType(349 MRI.getType(Reg).getScalarSizeInBits(), MIB);350 break;351 case TargetOpcode::G_TRUNC:352 case TargetOpcode::G_ADDRSPACE_CAST:353 case TargetOpcode::G_PTR_ADD:354 case TargetOpcode::COPY: {355 MachineOperand &Op = MI->getOperand(1);356 MachineInstr *Def = Op.isReg() ? MRI.getVRegDef(Op.getReg()) : nullptr;357 if (Def)358 SpvType = propagateSPIRVType(Def, GR, MRI, MIB);359 break;360 }361 default:362 break;363 }364 if (SpvType) {365 // check if the address space needs correction366 LLT RegType = MRI.getType(Reg);367 if (SpvType->getOpcode() == SPIRV::OpTypePointer &&368 RegType.isPointer() &&369 storageClassToAddressSpace(GR->getPointerStorageClass(SpvType)) !=370 RegType.getAddressSpace()) {371 const SPIRVSubtarget &ST =372 MI->getParent()->getParent()->getSubtarget<SPIRVSubtarget>();373 auto TSC = addressSpaceToStorageClass(RegType.getAddressSpace(), ST);374 SpvType = GR->changePointerStorageClass(SpvType, TSC, *MI);375 }376 GR->assignSPIRVTypeToVReg(SpvType, Reg, MIB.getMF());377 }378 if (!MRI.getRegClassOrNull(Reg))379 MRI.setRegClass(Reg, SpvType ? GR->getRegClass(SpvType)380 : &SPIRV::iIDRegClass);381 }382 }383 return SpvType;384}385 386// To support current approach and limitations wrt. bit width here we widen a387// scalar register with a bit width greater than 1 to valid sizes and cap it to388// 64 width.389static unsigned widenBitWidthToNextPow2(unsigned BitWidth) {390 if (BitWidth == 1)391 return 1; // No need to widen 1-bit values392 return std::min(std::max(1u << Log2_32_Ceil(BitWidth), 8u), 64u);393}394 395static void widenScalarType(Register Reg, MachineRegisterInfo &MRI) {396 LLT RegType = MRI.getType(Reg);397 if (!RegType.isScalar())398 return;399 unsigned CurrentWidth = RegType.getScalarSizeInBits();400 unsigned NewWidth = widenBitWidthToNextPow2(CurrentWidth);401 if (NewWidth != CurrentWidth)402 MRI.setType(Reg, LLT::scalar(NewWidth));403}404 405static void widenCImmType(MachineOperand &MOP) {406 const ConstantInt *CImmVal = MOP.getCImm();407 unsigned CurrentWidth = CImmVal->getBitWidth();408 unsigned NewWidth = widenBitWidthToNextPow2(CurrentWidth);409 if (NewWidth != CurrentWidth) {410 // Replace the immediate value with the widened version411 MOP.setCImm(ConstantInt::get(CImmVal->getType()->getContext(),412 CImmVal->getValue().zextOrTrunc(NewWidth)));413 }414}415 416static void setInsertPtAfterDef(MachineIRBuilder &MIB, MachineInstr *Def) {417 MachineBasicBlock &MBB = *Def->getParent();418 MachineBasicBlock::iterator DefIt =419 Def->getNextNode() ? Def->getNextNode()->getIterator() : MBB.end();420 // Skip all the PHI and debug instructions.421 while (DefIt != MBB.end() &&422 (DefIt->isPHI() || DefIt->isDebugOrPseudoInstr()))423 DefIt = std::next(DefIt);424 MIB.setInsertPt(MBB, DefIt);425}426 427namespace llvm {428void insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpvType,429 SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB,430 MachineRegisterInfo &MRI) {431 assert((Ty || SpvType) && "Either LLVM or SPIRV type is expected.");432 MachineInstr *Def = MRI.getVRegDef(Reg);433 setInsertPtAfterDef(MIB, Def);434 if (!SpvType)435 SpvType = GR->getOrCreateSPIRVType(Ty, MIB,436 SPIRV::AccessQualifier::ReadWrite, true);437 438 if (!isTypeFoldingSupported(Def->getOpcode())) {439 // No need to generate SPIRV::ASSIGN_TYPE pseudo-instruction440 if (!MRI.getRegClassOrNull(Reg))441 MRI.setRegClass(Reg, GR->getRegClass(SpvType));442 if (!MRI.getType(Reg).isValid())443 MRI.setType(Reg, GR->getRegType(SpvType));444 GR->assignSPIRVTypeToVReg(SpvType, Reg, MIB.getMF());445 return;446 }447 448 // Tablegen definition assumes SPIRV::ASSIGN_TYPE pseudo-instruction is449 // present after each auto-folded instruction to take a type reference from.450 Register NewReg = MRI.createGenericVirtualRegister(MRI.getType(Reg));451 const auto *RegClass = GR->getRegClass(SpvType);452 MRI.setRegClass(NewReg, RegClass);453 MRI.setRegClass(Reg, RegClass);454 455 GR->assignSPIRVTypeToVReg(SpvType, Reg, MIB.getMF());456 // This is to make it convenient for Legalizer to get the SPIRVType457 // when processing the actual MI (i.e. not pseudo one).458 GR->assignSPIRVTypeToVReg(SpvType, NewReg, MIB.getMF());459 // Copy MIFlags from Def to ASSIGN_TYPE instruction. It's required to keep460 // the flags after instruction selection.461 const uint32_t Flags = Def->getFlags();462 MIB.buildInstr(SPIRV::ASSIGN_TYPE)463 .addDef(Reg)464 .addUse(NewReg)465 .addUse(GR->getSPIRVTypeID(SpvType))466 .setMIFlags(Flags);467 for (unsigned I = 0, E = Def->getNumDefs(); I != E; ++I) {468 MachineOperand &MO = Def->getOperand(I);469 if (MO.getReg() == Reg) {470 MO.setReg(NewReg);471 break;472 }473 }474}475 476void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,477 MachineRegisterInfo &MRI, SPIRVGlobalRegistry *GR,478 SPIRVType *KnownResType) {479 MIB.setInsertPt(*MI.getParent(), MI.getIterator());480 for (auto &Op : MI.operands()) {481 if (!Op.isReg() || Op.isDef())482 continue;483 Register OpReg = Op.getReg();484 SPIRVType *SpvType = GR->getSPIRVTypeForVReg(OpReg);485 if (!SpvType && KnownResType) {486 SpvType = KnownResType;487 GR->assignSPIRVTypeToVReg(KnownResType, OpReg, *MI.getMF());488 }489 assert(SpvType);490 if (!MRI.getRegClassOrNull(OpReg))491 MRI.setRegClass(OpReg, GR->getRegClass(SpvType));492 if (!MRI.getType(OpReg).isValid())493 MRI.setType(OpReg, GR->getRegType(SpvType));494 }495}496} // namespace llvm497 498static void499generateAssignInstrs(MachineFunction &MF, SPIRVGlobalRegistry *GR,500 MachineIRBuilder MIB,501 DenseMap<MachineInstr *, Type *> &TargetExtConstTypes) {502 // Get access to information about available extensions503 const SPIRVSubtarget *ST =504 static_cast<const SPIRVSubtarget *>(&MIB.getMF().getSubtarget());505 506 MachineRegisterInfo &MRI = MF.getRegInfo();507 SmallVector<MachineInstr *, 10> ToErase;508 DenseMap<MachineInstr *, Register> RegsAlreadyAddedToDT;509 510 bool IsExtendedInts =511 ST->canUseExtension(512 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) ||513 ST->canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions) ||514 ST->canUseExtension(SPIRV::Extension::SPV_INTEL_int4);515 516 for (MachineBasicBlock *MBB : post_order(&MF)) {517 if (MBB->empty())518 continue;519 520 bool ReachedBegin = false;521 for (auto MII = std::prev(MBB->end()), Begin = MBB->begin();522 !ReachedBegin;) {523 MachineInstr &MI = *MII;524 unsigned MIOp = MI.getOpcode();525 526 if (!IsExtendedInts) {527 // validate bit width of scalar registers and constant immediates528 for (auto &MOP : MI.operands()) {529 if (MOP.isReg())530 widenScalarType(MOP.getReg(), MRI);531 else if (MOP.isCImm())532 widenCImmType(MOP);533 }534 }535 536 if (isSpvIntrinsic(MI, Intrinsic::spv_assign_ptr_type)) {537 Register Reg = MI.getOperand(1).getReg();538 MIB.setInsertPt(*MI.getParent(), MI.getIterator());539 Type *ElementTy = getMDOperandAsType(MI.getOperand(2).getMetadata(), 0);540 SPIRVType *AssignedPtrType = GR->getOrCreateSPIRVPointerType(541 ElementTy, MI,542 addressSpaceToStorageClass(MI.getOperand(3).getImm(), *ST));543 MachineInstr *Def = MRI.getVRegDef(Reg);544 assert(Def && "Expecting an instruction that defines the register");545 // G_GLOBAL_VALUE already has type info.546 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE &&547 Def->getOpcode() != SPIRV::ASSIGN_TYPE)548 insertAssignInstr(Reg, nullptr, AssignedPtrType, GR, MIB,549 MF.getRegInfo());550 ToErase.push_back(&MI);551 } else if (isSpvIntrinsic(MI, Intrinsic::spv_assign_type)) {552 Register Reg = MI.getOperand(1).getReg();553 Type *Ty = getMDOperandAsType(MI.getOperand(2).getMetadata(), 0);554 MachineInstr *Def = MRI.getVRegDef(Reg);555 assert(Def && "Expecting an instruction that defines the register");556 // G_GLOBAL_VALUE already has type info.557 if (Def->getOpcode() != TargetOpcode::G_GLOBAL_VALUE &&558 Def->getOpcode() != SPIRV::ASSIGN_TYPE)559 insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MF.getRegInfo());560 ToErase.push_back(&MI);561 } else if (MIOp == TargetOpcode::FAKE_USE && MI.getNumOperands() > 0) {562 MachineInstr *MdMI = MI.getPrevNode();563 if (MdMI && isSpvIntrinsic(*MdMI, Intrinsic::spv_value_md)) {564 // It's an internal service info from before IRTranslator passes.565 MachineInstr *Def = getVRegDef(MRI, MI.getOperand(0).getReg());566 for (unsigned I = 1, E = MI.getNumOperands(); I != E && Def; ++I)567 if (getVRegDef(MRI, MI.getOperand(I).getReg()) != Def)568 Def = nullptr;569 if (Def) {570 const MDNode *MD = MdMI->getOperand(1).getMetadata();571 StringRef ValueName =572 cast<MDString>(MD->getOperand(1))->getString();573 const MDNode *TypeMD = cast<MDNode>(MD->getOperand(0));574 Type *ValueTy = getMDOperandAsType(TypeMD, 0);575 GR->addValueAttrs(Def, std::make_pair(ValueTy, ValueName.str()));576 }577 ToErase.push_back(MdMI);578 }579 ToErase.push_back(&MI);580 } else if (MIOp == TargetOpcode::G_CONSTANT ||581 MIOp == TargetOpcode::G_FCONSTANT ||582 MIOp == TargetOpcode::G_BUILD_VECTOR) {583 // %rc = G_CONSTANT ty Val584 // ===>585 // %cty = OpType* ty586 // %rctmp = G_CONSTANT ty Val587 // %rc = ASSIGN_TYPE %rctmp, %cty588 Register Reg = MI.getOperand(0).getReg();589 bool NeedAssignType = true;590 if (MRI.hasOneUse(Reg)) {591 MachineInstr &UseMI = *MRI.use_instr_begin(Reg);592 if (isSpvIntrinsic(UseMI, Intrinsic::spv_assign_type) ||593 isSpvIntrinsic(UseMI, Intrinsic::spv_assign_name))594 continue;595 if (UseMI.getOpcode() == SPIRV::ASSIGN_TYPE)596 NeedAssignType = false;597 }598 Type *Ty = nullptr;599 if (MIOp == TargetOpcode::G_CONSTANT) {600 auto TargetExtIt = TargetExtConstTypes.find(&MI);601 Ty = TargetExtIt == TargetExtConstTypes.end()602 ? MI.getOperand(1).getCImm()->getType()603 : TargetExtIt->second;604 const ConstantInt *OpCI = MI.getOperand(1).getCImm();605 // TODO: we may wish to analyze here if OpCI is zero and LLT RegType =606 // MRI.getType(Reg); RegType.isPointer() is true, so that we observe607 // at this point not i64/i32 constant but null pointer in the608 // corresponding address space of RegType.getAddressSpace(). This may609 // help to successfully validate the case when a OpConstantComposite's610 // constituent has type that does not match Result Type of611 // OpConstantComposite (see, for example,612 // pointers/PtrCast-null-in-OpSpecConstantOp.ll).613 Register PrimaryReg = GR->find(OpCI, &MF);614 if (!PrimaryReg.isValid()) {615 GR->add(OpCI, &MI);616 } else if (PrimaryReg != Reg &&617 MRI.getType(Reg) == MRI.getType(PrimaryReg)) {618 auto *RCReg = MRI.getRegClassOrNull(Reg);619 auto *RCPrimary = MRI.getRegClassOrNull(PrimaryReg);620 if (!RCReg || RCPrimary == RCReg) {621 RegsAlreadyAddedToDT[&MI] = PrimaryReg;622 ToErase.push_back(&MI);623 NeedAssignType = false;624 }625 }626 } else if (MIOp == TargetOpcode::G_FCONSTANT) {627 Ty = MI.getOperand(1).getFPImm()->getType();628 } else {629 assert(MIOp == TargetOpcode::G_BUILD_VECTOR);630 Type *ElemTy = nullptr;631 MachineInstr *ElemMI = MRI.getVRegDef(MI.getOperand(1).getReg());632 assert(ElemMI);633 634 if (ElemMI->getOpcode() == TargetOpcode::G_CONSTANT) {635 ElemTy = ElemMI->getOperand(1).getCImm()->getType();636 } else if (ElemMI->getOpcode() == TargetOpcode::G_FCONSTANT) {637 ElemTy = ElemMI->getOperand(1).getFPImm()->getType();638 } else {639 if (const SPIRVType *ElemSpvType =640 GR->getSPIRVTypeForVReg(MI.getOperand(1).getReg(), &MF))641 ElemTy = const_cast<Type *>(GR->getTypeForSPIRVType(ElemSpvType));642 if (!ElemTy) {643 // There may be a case when we already know Reg's type.644 MachineInstr *NextMI = MI.getNextNode();645 if (!NextMI || NextMI->getOpcode() != SPIRV::ASSIGN_TYPE ||646 NextMI->getOperand(1).getReg() != Reg)647 llvm_unreachable("Unexpected opcode");648 }649 }650 if (ElemTy)651 Ty = VectorType::get(652 ElemTy, MI.getNumExplicitOperands() - MI.getNumExplicitDefs(),653 false);654 else655 NeedAssignType = false;656 }657 if (NeedAssignType)658 insertAssignInstr(Reg, Ty, nullptr, GR, MIB, MRI);659 } else if (MIOp == TargetOpcode::G_GLOBAL_VALUE) {660 propagateSPIRVType(&MI, GR, MRI, MIB);661 }662 663 if (MII == Begin)664 ReachedBegin = true;665 else666 --MII;667 }668 }669 for (MachineInstr *MI : ToErase) {670 auto It = RegsAlreadyAddedToDT.find(MI);671 if (It != RegsAlreadyAddedToDT.end())672 MRI.replaceRegWith(MI->getOperand(0).getReg(), It->second);673 GR->invalidateMachineInstr(MI);674 MI->eraseFromParent();675 }676 677 // Address the case when IRTranslator introduces instructions with new678 // registers without SPIRVType associated.679 for (MachineBasicBlock &MBB : MF) {680 for (MachineInstr &MI : MBB) {681 switch (MI.getOpcode()) {682 case TargetOpcode::G_TRUNC:683 case TargetOpcode::G_ANYEXT:684 case TargetOpcode::G_SEXT:685 case TargetOpcode::G_ZEXT:686 case TargetOpcode::G_PTRTOINT:687 case TargetOpcode::COPY:688 case TargetOpcode::G_ADDRSPACE_CAST:689 propagateSPIRVType(&MI, GR, MRI, MIB);690 break;691 }692 }693 }694}695 696static void processInstrsWithTypeFolding(MachineFunction &MF,697 SPIRVGlobalRegistry *GR,698 MachineIRBuilder MIB) {699 MachineRegisterInfo &MRI = MF.getRegInfo();700 for (MachineBasicBlock &MBB : MF)701 for (MachineInstr &MI : MBB)702 if (isTypeFoldingSupported(MI.getOpcode()))703 processInstr(MI, MIB, MRI, GR, nullptr);704}705 706static Register707collectInlineAsmInstrOperands(MachineInstr *MI,708 SmallVector<unsigned, 4> *Ops = nullptr) {709 Register DefReg;710 unsigned StartOp = InlineAsm::MIOp_FirstOperand,711 AsmDescOp = InlineAsm::MIOp_FirstOperand;712 for (unsigned Idx = StartOp, MISz = MI->getNumOperands(); Idx != MISz;713 ++Idx) {714 const MachineOperand &MO = MI->getOperand(Idx);715 if (MO.isMetadata())716 continue;717 if (Idx == AsmDescOp && MO.isImm()) {718 // compute the index of the next operand descriptor719 const InlineAsm::Flag F(MO.getImm());720 AsmDescOp += 1 + F.getNumOperandRegisters();721 continue;722 }723 if (MO.isReg() && MO.isDef()) {724 if (!Ops)725 return MO.getReg();726 else727 DefReg = MO.getReg();728 } else if (Ops) {729 Ops->push_back(Idx);730 }731 }732 return DefReg;733}734 735static void736insertInlineAsmProcess(MachineFunction &MF, SPIRVGlobalRegistry *GR,737 const SPIRVSubtarget &ST, MachineIRBuilder MIRBuilder,738 const SmallVector<MachineInstr *> &ToProcess) {739 MachineRegisterInfo &MRI = MF.getRegInfo();740 Register AsmTargetReg;741 for (unsigned i = 0, Sz = ToProcess.size(); i + 1 < Sz; i += 2) {742 MachineInstr *I1 = ToProcess[i], *I2 = ToProcess[i + 1];743 assert(isSpvIntrinsic(*I1, Intrinsic::spv_inline_asm) && I2->isInlineAsm());744 MIRBuilder.setInsertPt(*I2->getParent(), *I2);745 746 if (!AsmTargetReg.isValid()) {747 // define vendor specific assembly target or dialect748 AsmTargetReg = MRI.createGenericVirtualRegister(LLT::scalar(32));749 MRI.setRegClass(AsmTargetReg, &SPIRV::iIDRegClass);750 auto AsmTargetMIB =751 MIRBuilder.buildInstr(SPIRV::OpAsmTargetINTEL).addDef(AsmTargetReg);752 addStringImm(ST.getTargetTripleAsStr(), AsmTargetMIB);753 GR->add(AsmTargetMIB.getInstr(), AsmTargetMIB);754 }755 756 // create types757 const MDNode *IAMD = I1->getOperand(1).getMetadata();758 FunctionType *FTy = cast<FunctionType>(getMDOperandAsType(IAMD, 0));759 SmallVector<SPIRVType *, 4> ArgTypes;760 for (const auto &ArgTy : FTy->params())761 ArgTypes.push_back(GR->getOrCreateSPIRVType(762 ArgTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true));763 SPIRVType *RetType =764 GR->getOrCreateSPIRVType(FTy->getReturnType(), MIRBuilder,765 SPIRV::AccessQualifier::ReadWrite, true);766 SPIRVType *FuncType = GR->getOrCreateOpTypeFunctionWithArgs(767 FTy, RetType, ArgTypes, MIRBuilder);768 769 // define vendor specific assembly instructions string770 Register AsmReg = MRI.createGenericVirtualRegister(LLT::scalar(32));771 MRI.setRegClass(AsmReg, &SPIRV::iIDRegClass);772 auto AsmMIB = MIRBuilder.buildInstr(SPIRV::OpAsmINTEL)773 .addDef(AsmReg)774 .addUse(GR->getSPIRVTypeID(RetType))775 .addUse(GR->getSPIRVTypeID(FuncType))776 .addUse(AsmTargetReg);777 // inline asm string:778 addStringImm(I2->getOperand(InlineAsm::MIOp_AsmString).getSymbolName(),779 AsmMIB);780 // inline asm constraint string:781 addStringImm(cast<MDString>(I1->getOperand(2).getMetadata()->getOperand(0))782 ->getString(),783 AsmMIB);784 GR->add(AsmMIB.getInstr(), AsmMIB);785 786 // calls the inline assembly instruction787 unsigned ExtraInfo = I2->getOperand(InlineAsm::MIOp_ExtraInfo).getImm();788 if (ExtraInfo & InlineAsm::Extra_HasSideEffects)789 MIRBuilder.buildInstr(SPIRV::OpDecorate)790 .addUse(AsmReg)791 .addImm(static_cast<uint32_t>(SPIRV::Decoration::SideEffectsINTEL));792 793 Register DefReg = collectInlineAsmInstrOperands(I2);794 if (!DefReg.isValid()) {795 DefReg = MRI.createGenericVirtualRegister(LLT::scalar(32));796 MRI.setRegClass(DefReg, &SPIRV::iIDRegClass);797 SPIRVType *VoidType = GR->getOrCreateSPIRVType(798 Type::getVoidTy(MF.getFunction().getContext()), MIRBuilder,799 SPIRV::AccessQualifier::ReadWrite, true);800 GR->assignSPIRVTypeToVReg(VoidType, DefReg, MF);801 }802 803 auto AsmCall = MIRBuilder.buildInstr(SPIRV::OpAsmCallINTEL)804 .addDef(DefReg)805 .addUse(GR->getSPIRVTypeID(RetType))806 .addUse(AsmReg);807 for (unsigned IntrIdx = 3; IntrIdx < I1->getNumOperands(); ++IntrIdx)808 AsmCall.addUse(I1->getOperand(IntrIdx).getReg());809 }810 for (MachineInstr *MI : ToProcess) {811 GR->invalidateMachineInstr(MI);812 MI->eraseFromParent();813 }814}815 816static void insertInlineAsm(MachineFunction &MF, SPIRVGlobalRegistry *GR,817 const SPIRVSubtarget &ST,818 MachineIRBuilder MIRBuilder) {819 SmallVector<MachineInstr *> ToProcess;820 for (MachineBasicBlock &MBB : MF) {821 for (MachineInstr &MI : MBB) {822 if (isSpvIntrinsic(MI, Intrinsic::spv_inline_asm) ||823 MI.getOpcode() == TargetOpcode::INLINEASM)824 ToProcess.push_back(&MI);825 }826 }827 if (ToProcess.size() == 0)828 return;829 830 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_inline_assembly))831 report_fatal_error("Inline assembly instructions require the "832 "following SPIR-V extension: SPV_INTEL_inline_assembly",833 false);834 835 insertInlineAsmProcess(MF, GR, ST, MIRBuilder, ToProcess);836}837 838static uint32_t convertFloatToSPIRVWord(float F) {839 union {840 float F;841 uint32_t Spir;842 } FPMaxError;843 FPMaxError.F = F;844 return FPMaxError.Spir;845}846 847static void insertSpirvDecorations(MachineFunction &MF, SPIRVGlobalRegistry *GR,848 MachineIRBuilder MIB) {849 const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(MIB.getMF().getSubtarget());850 SmallVector<MachineInstr *, 10> ToErase;851 for (MachineBasicBlock &MBB : MF) {852 for (MachineInstr &MI : MBB) {853 if (!isSpvIntrinsic(MI, Intrinsic::spv_assign_decoration) &&854 !isSpvIntrinsic(MI, Intrinsic::spv_assign_aliasing_decoration) &&855 !isSpvIntrinsic(MI, Intrinsic::spv_assign_fpmaxerror_decoration))856 continue;857 MIB.setInsertPt(*MI.getParent(), MI.getNextNode());858 if (isSpvIntrinsic(MI, Intrinsic::spv_assign_decoration)) {859 buildOpSpirvDecorations(MI.getOperand(1).getReg(), MIB,860 MI.getOperand(2).getMetadata(), ST);861 } else if (isSpvIntrinsic(MI,862 Intrinsic::spv_assign_fpmaxerror_decoration)) {863 ConstantFP *OpV = mdconst::dyn_extract<ConstantFP>(864 MI.getOperand(2).getMetadata()->getOperand(0));865 uint32_t OpValue =866 convertFloatToSPIRVWord(OpV->getValueAPF().convertToFloat());867 868 buildOpDecorate(MI.getOperand(1).getReg(), MIB,869 SPIRV::Decoration::FPMaxErrorDecorationINTEL,870 {OpValue});871 } else {872 GR->buildMemAliasingOpDecorate(MI.getOperand(1).getReg(), MIB,873 MI.getOperand(2).getImm(),874 MI.getOperand(3).getMetadata());875 }876 877 ToErase.push_back(&MI);878 }879 }880 for (MachineInstr *MI : ToErase) {881 GR->invalidateMachineInstr(MI);882 MI->eraseFromParent();883 }884}885 886// LLVM allows the switches to use registers as cases, while SPIR-V required887// those to be immediate values. This function replaces such operands with the888// equivalent immediate constant.889static void processSwitchesConstants(MachineFunction &MF,890 SPIRVGlobalRegistry *GR,891 MachineIRBuilder MIB) {892 MachineRegisterInfo &MRI = MF.getRegInfo();893 for (MachineBasicBlock &MBB : MF) {894 for (MachineInstr &MI : MBB) {895 if (!isSpvIntrinsic(MI, Intrinsic::spv_switch))896 continue;897 898 SmallVector<MachineOperand, 8> NewOperands;899 NewOperands.push_back(MI.getOperand(0)); // Opcode900 NewOperands.push_back(MI.getOperand(1)); // Condition901 NewOperands.push_back(MI.getOperand(2)); // Default902 for (unsigned i = 3; i < MI.getNumOperands(); i += 2) {903 Register Reg = MI.getOperand(i).getReg();904 MachineInstr *ConstInstr = getDefInstrMaybeConstant(Reg, &MRI);905 NewOperands.push_back(906 MachineOperand::CreateCImm(ConstInstr->getOperand(1).getCImm()));907 908 NewOperands.push_back(MI.getOperand(i + 1));909 }910 911 assert(MI.getNumOperands() == NewOperands.size());912 while (MI.getNumOperands() > 0)913 MI.removeOperand(0);914 for (auto &MO : NewOperands)915 MI.addOperand(MO);916 }917 }918}919 920// Some instructions are used during CodeGen but should never be emitted.921// Cleaning up those.922static void cleanupHelperInstructions(MachineFunction &MF,923 SPIRVGlobalRegistry *GR) {924 SmallVector<MachineInstr *, 8> ToEraseMI;925 for (MachineBasicBlock &MBB : MF) {926 for (MachineInstr &MI : MBB) {927 if (isSpvIntrinsic(MI, Intrinsic::spv_track_constant) ||928 MI.getOpcode() == TargetOpcode::G_BRINDIRECT)929 ToEraseMI.push_back(&MI);930 }931 }932 933 for (MachineInstr *MI : ToEraseMI) {934 GR->invalidateMachineInstr(MI);935 MI->eraseFromParent();936 }937}938 939// Find all usages of G_BLOCK_ADDR in our intrinsics and replace those940// operands/registers by the actual MBB it references.941static void processBlockAddr(MachineFunction &MF, SPIRVGlobalRegistry *GR,942 MachineIRBuilder MIB) {943 // Gather the reverse-mapping BB -> MBB.944 DenseMap<const BasicBlock *, MachineBasicBlock *> BB2MBB;945 for (MachineBasicBlock &MBB : MF)946 BB2MBB[MBB.getBasicBlock()] = &MBB;947 948 // Gather instructions requiring patching. For now, only those can use949 // G_BLOCK_ADDR.950 SmallVector<MachineInstr *, 8> InstructionsToPatch;951 for (MachineBasicBlock &MBB : MF) {952 for (MachineInstr &MI : MBB) {953 if (isSpvIntrinsic(MI, Intrinsic::spv_switch) ||954 isSpvIntrinsic(MI, Intrinsic::spv_loop_merge) ||955 isSpvIntrinsic(MI, Intrinsic::spv_selection_merge))956 InstructionsToPatch.push_back(&MI);957 }958 }959 960 // For each instruction to fix, we replace all the G_BLOCK_ADDR operands by961 // the actual MBB it references. Once those references have been updated, we962 // can cleanup remaining G_BLOCK_ADDR references.963 SmallPtrSet<MachineBasicBlock *, 8> ClearAddressTaken;964 SmallPtrSet<MachineInstr *, 8> ToEraseMI;965 MachineRegisterInfo &MRI = MF.getRegInfo();966 for (MachineInstr *MI : InstructionsToPatch) {967 SmallVector<MachineOperand, 8> NewOps;968 for (unsigned i = 0; i < MI->getNumOperands(); ++i) {969 // The operand is not a register, keep as-is.970 if (!MI->getOperand(i).isReg()) {971 NewOps.push_back(MI->getOperand(i));972 continue;973 }974 975 Register Reg = MI->getOperand(i).getReg();976 MachineInstr *BuildMBB = MRI.getVRegDef(Reg);977 // The register is not the result of G_BLOCK_ADDR, keep as-is.978 if (!BuildMBB || BuildMBB->getOpcode() != TargetOpcode::G_BLOCK_ADDR) {979 NewOps.push_back(MI->getOperand(i));980 continue;981 }982 983 assert(BuildMBB && BuildMBB->getOpcode() == TargetOpcode::G_BLOCK_ADDR &&984 BuildMBB->getOperand(1).isBlockAddress() &&985 BuildMBB->getOperand(1).getBlockAddress());986 BasicBlock *BB =987 BuildMBB->getOperand(1).getBlockAddress()->getBasicBlock();988 auto It = BB2MBB.find(BB);989 if (It == BB2MBB.end())990 report_fatal_error("cannot find a machine basic block by a basic block "991 "in a switch statement");992 MachineBasicBlock *ReferencedBlock = It->second;993 NewOps.push_back(MachineOperand::CreateMBB(ReferencedBlock));994 995 ClearAddressTaken.insert(ReferencedBlock);996 ToEraseMI.insert(BuildMBB);997 }998 999 // Replace the operands.1000 assert(MI->getNumOperands() == NewOps.size());1001 while (MI->getNumOperands() > 0)1002 MI->removeOperand(0);1003 for (auto &MO : NewOps)1004 MI->addOperand(MO);1005 1006 if (MachineInstr *Next = MI->getNextNode()) {1007 if (isSpvIntrinsic(*Next, Intrinsic::spv_track_constant)) {1008 ToEraseMI.insert(Next);1009 Next = MI->getNextNode();1010 }1011 if (Next && Next->getOpcode() == TargetOpcode::G_BRINDIRECT)1012 ToEraseMI.insert(Next);1013 }1014 }1015 1016 // BlockAddress operands were used to keep information between passes,1017 // let's undo the "address taken" status to reflect that Succ doesn't1018 // actually correspond to an IR-level basic block.1019 for (MachineBasicBlock *Succ : ClearAddressTaken)1020 Succ->setAddressTakenIRBlock(nullptr);1021 1022 // If we just delete G_BLOCK_ADDR instructions with BlockAddress operands,1023 // this leaves their BasicBlock counterparts in a "address taken" status. This1024 // would make AsmPrinter to generate a series of unneeded labels of a "Address1025 // of block that was removed by CodeGen" kind. Let's first ensure that we1026 // don't have a dangling BlockAddress constants by zapping the BlockAddress1027 // nodes, and only after that proceed with erasing G_BLOCK_ADDR instructions.1028 Constant *Replacement =1029 ConstantInt::get(Type::getInt32Ty(MF.getFunction().getContext()), 1);1030 for (MachineInstr *BlockAddrI : ToEraseMI) {1031 if (BlockAddrI->getOpcode() == TargetOpcode::G_BLOCK_ADDR) {1032 BlockAddress *BA = const_cast<BlockAddress *>(1033 BlockAddrI->getOperand(1).getBlockAddress());1034 BA->replaceAllUsesWith(1035 ConstantExpr::getIntToPtr(Replacement, BA->getType()));1036 BA->destroyConstant();1037 }1038 GR->invalidateMachineInstr(BlockAddrI);1039 BlockAddrI->eraseFromParent();1040 }1041}1042 1043static bool isImplicitFallthrough(MachineBasicBlock &MBB) {1044 if (MBB.empty())1045 return true;1046 1047 // Branching SPIR-V intrinsics are not detected by this generic method.1048 // Thus, we can only trust negative result.1049 if (!MBB.canFallThrough())1050 return false;1051 1052 // Otherwise, we must manually check if we have a SPIR-V intrinsic which1053 // prevent an implicit fallthrough.1054 for (MachineBasicBlock::reverse_iterator It = MBB.rbegin(), E = MBB.rend();1055 It != E; ++It) {1056 if (isSpvIntrinsic(*It, Intrinsic::spv_switch))1057 return false;1058 }1059 return true;1060}1061 1062static void removeImplicitFallthroughs(MachineFunction &MF,1063 MachineIRBuilder MIB) {1064 // It is valid for MachineBasicBlocks to not finish with a branch instruction.1065 // In such cases, they will simply fallthrough their immediate successor.1066 for (MachineBasicBlock &MBB : MF) {1067 if (!isImplicitFallthrough(MBB))1068 continue;1069 1070 assert(MBB.succ_size() == 1);1071 MIB.setInsertPt(MBB, MBB.end());1072 MIB.buildBr(**MBB.successors().begin());1073 }1074}1075 1076bool SPIRVPreLegalizer::runOnMachineFunction(MachineFunction &MF) {1077 // Initialize the type registry.1078 const SPIRVSubtarget &ST = MF.getSubtarget<SPIRVSubtarget>();1079 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();1080 GR->setCurrentFunc(MF);1081 MachineIRBuilder MIB(MF);1082 // a registry of target extension constants1083 DenseMap<MachineInstr *, Type *> TargetExtConstTypes;1084 // to keep record of tracked constants1085 addConstantsToTrack(MF, GR, ST, TargetExtConstTypes);1086 foldConstantsIntoIntrinsics(MF, GR, MIB);1087 insertBitcasts(MF, GR, MIB);1088 generateAssignInstrs(MF, GR, MIB, TargetExtConstTypes);1089 1090 processSwitchesConstants(MF, GR, MIB);1091 processBlockAddr(MF, GR, MIB);1092 cleanupHelperInstructions(MF, GR);1093 1094 processInstrsWithTypeFolding(MF, GR, MIB);1095 removeImplicitFallthroughs(MF, MIB);1096 insertSpirvDecorations(MF, GR, MIB);1097 insertInlineAsm(MF, GR, ST, MIB);1098 lowerBitcasts(MF, GR, MIB);1099 1100 return true;1101}1102 1103INITIALIZE_PASS(SPIRVPreLegalizer, DEBUG_TYPE, "SPIRV pre legalizer", false,1104 false)1105 1106char SPIRVPreLegalizer::ID = 0;1107 1108FunctionPass *llvm::createSPIRVPreLegalizerPass() {1109 return new SPIRVPreLegalizer();1110}1111