brintos / llvm-project-archived public Read only

1//===-- SPIRVGlobalRegistry.cpp - SPIR-V Global Registry --------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains the implementation of the SPIRVGlobalRegistry class,10// which is used to maintain rich type information required for SPIR-V even11// after lowering from LLVM IR to GMIR. It can convert an llvm::Type into12// an OpTypeXXX instruction, and map it to a virtual register. Also it builds13// and supports consistency of constants and global variables.14//15//===----------------------------------------------------------------------===//16 17#include "SPIRVGlobalRegistry.h"18#include "SPIRV.h"19#include "SPIRVBuiltins.h"20#include "SPIRVSubtarget.h"21#include "SPIRVUtils.h"22#include "llvm/ADT/APInt.h"23#include "llvm/IR/Constants.h"24#include "llvm/IR/DiagnosticInfo.h"25#include "llvm/IR/Function.h"26#include "llvm/IR/IntrinsicInst.h"27#include "llvm/IR/Intrinsics.h"28#include "llvm/IR/IntrinsicsSPIRV.h"29#include "llvm/IR/Type.h"30#include "llvm/Support/Casting.h"31#include "llvm/Support/MathExtras.h"32#include <cassert>33#include <functional>34 35using namespace llvm;36 37static bool allowEmitFakeUse(const Value *Arg) {38  if (isSpvIntrinsic(Arg))39    return false;40  if (isa<AtomicCmpXchgInst, InsertValueInst, UndefValue>(Arg))41    return false;42  if (const auto *LI = dyn_cast<LoadInst>(Arg))43    if (LI->getType()->isAggregateType())44      return false;45  return true;46}47 48static unsigned typeToAddressSpace(const Type *Ty) {49  if (auto PType = dyn_cast<TypedPointerType>(Ty))50    return PType->getAddressSpace();51  if (auto PType = dyn_cast<PointerType>(Ty))52    return PType->getAddressSpace();53  if (auto *ExtTy = dyn_cast<TargetExtType>(Ty);54      ExtTy && isTypedPointerWrapper(ExtTy))55    return ExtTy->getIntParameter(0);56  reportFatalInternalError("Unable to convert LLVM type to SPIRVType");57}58 59static bool60storageClassRequiresExplictLayout(SPIRV::StorageClass::StorageClass SC) {61  switch (SC) {62  case SPIRV::StorageClass::Uniform:63  case SPIRV::StorageClass::PushConstant:64  case SPIRV::StorageClass::StorageBuffer:65  case SPIRV::StorageClass::PhysicalStorageBufferEXT:66    return true;67  case SPIRV::StorageClass::UniformConstant:68  case SPIRV::StorageClass::Input:69  case SPIRV::StorageClass::Output:70  case SPIRV::StorageClass::Workgroup:71  case SPIRV::StorageClass::CrossWorkgroup:72  case SPIRV::StorageClass::Private:73  case SPIRV::StorageClass::Function:74  case SPIRV::StorageClass::Generic:75  case SPIRV::StorageClass::AtomicCounter:76  case SPIRV::StorageClass::Image:77  case SPIRV::StorageClass::CallableDataNV:78  case SPIRV::StorageClass::IncomingCallableDataNV:79  case SPIRV::StorageClass::RayPayloadNV:80  case SPIRV::StorageClass::HitAttributeNV:81  case SPIRV::StorageClass::IncomingRayPayloadNV:82  case SPIRV::StorageClass::ShaderRecordBufferNV:83  case SPIRV::StorageClass::CodeSectionINTEL:84  case SPIRV::StorageClass::DeviceOnlyINTEL:85  case SPIRV::StorageClass::HostOnlyINTEL:86    return false;87  }88  llvm_unreachable("Unknown SPIRV::StorageClass enum");89}90 91SPIRVGlobalRegistry::SPIRVGlobalRegistry(unsigned PointerSize)92    : PointerSize(PointerSize), Bound(0), CurMF(nullptr) {}93 94SPIRVType *SPIRVGlobalRegistry::assignIntTypeToVReg(unsigned BitWidth,95                                                    Register VReg,96                                                    MachineInstr &I,97                                                    const SPIRVInstrInfo &TII) {98  SPIRVType *SpirvType = getOrCreateSPIRVIntegerType(BitWidth, I, TII);99  assignSPIRVTypeToVReg(SpirvType, VReg, *CurMF);100  return SpirvType;101}102 103SPIRVType *104SPIRVGlobalRegistry::assignFloatTypeToVReg(unsigned BitWidth, Register VReg,105                                           MachineInstr &I,106                                           const SPIRVInstrInfo &TII) {107  SPIRVType *SpirvType = getOrCreateSPIRVFloatType(BitWidth, I, TII);108  assignSPIRVTypeToVReg(SpirvType, VReg, *CurMF);109  return SpirvType;110}111 112SPIRVType *SPIRVGlobalRegistry::assignVectTypeToVReg(113    SPIRVType *BaseType, unsigned NumElements, Register VReg, MachineInstr &I,114    const SPIRVInstrInfo &TII) {115  SPIRVType *SpirvType =116      getOrCreateSPIRVVectorType(BaseType, NumElements, I, TII);117  assignSPIRVTypeToVReg(SpirvType, VReg, *CurMF);118  return SpirvType;119}120 121SPIRVType *SPIRVGlobalRegistry::assignTypeToVReg(122    const Type *Type, Register VReg, MachineIRBuilder &MIRBuilder,123    SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {124  SPIRVType *SpirvType =125      getOrCreateSPIRVType(Type, MIRBuilder, AccessQual, EmitIR);126  assignSPIRVTypeToVReg(SpirvType, VReg, MIRBuilder.getMF());127  return SpirvType;128}129 130void SPIRVGlobalRegistry::assignSPIRVTypeToVReg(SPIRVType *SpirvType,131                                                Register VReg,132                                                const MachineFunction &MF) {133  VRegToTypeMap[&MF][VReg] = SpirvType;134}135 136static Register createTypeVReg(MachineRegisterInfo &MRI) {137  auto Res = MRI.createGenericVirtualRegister(LLT::scalar(64));138  MRI.setRegClass(Res, &SPIRV::TYPERegClass);139  return Res;140}141 142inline Register createTypeVReg(MachineIRBuilder &MIRBuilder) {143  return createTypeVReg(MIRBuilder.getMF().getRegInfo());144}145 146SPIRVType *SPIRVGlobalRegistry::getOpTypeBool(MachineIRBuilder &MIRBuilder) {147  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {148    return MIRBuilder.buildInstr(SPIRV::OpTypeBool)149        .addDef(createTypeVReg(MIRBuilder));150  });151}152 153unsigned SPIRVGlobalRegistry::adjustOpTypeIntWidth(unsigned Width) const {154  if (Width > 64)155    report_fatal_error("Unsupported integer width!");156  const SPIRVSubtarget &ST = cast<SPIRVSubtarget>(CurMF->getSubtarget());157  if (ST.canUseExtension(158          SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers) ||159      ST.canUseExtension(SPIRV::Extension::SPV_INTEL_int4))160    return Width;161  if (Width <= 8)162    Width = 8;163  else if (Width <= 16)164    Width = 16;165  else if (Width <= 32)166    Width = 32;167  else168    Width = 64;169  return Width;170}171 172SPIRVType *SPIRVGlobalRegistry::getOpTypeInt(unsigned Width,173                                             MachineIRBuilder &MIRBuilder,174                                             bool IsSigned) {175  Width = adjustOpTypeIntWidth(Width);176  const SPIRVSubtarget &ST =177      cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget());178  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {179    if (Width == 4 && ST.canUseExtension(SPIRV::Extension::SPV_INTEL_int4)) {180      MIRBuilder.buildInstr(SPIRV::OpExtension)181          .addImm(SPIRV::Extension::SPV_INTEL_int4);182      MIRBuilder.buildInstr(SPIRV::OpCapability)183          .addImm(SPIRV::Capability::Int4TypeINTEL);184    } else if ((!isPowerOf2_32(Width) || Width < 8) &&185               ST.canUseExtension(186                   SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers)) {187      MIRBuilder.buildInstr(SPIRV::OpExtension)188          .addImm(SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);189      MIRBuilder.buildInstr(SPIRV::OpCapability)190          .addImm(SPIRV::Capability::ArbitraryPrecisionIntegersALTERA);191    }192    return MIRBuilder.buildInstr(SPIRV::OpTypeInt)193        .addDef(createTypeVReg(MIRBuilder))194        .addImm(Width)195        .addImm(IsSigned ? 1 : 0);196  });197}198 199SPIRVType *SPIRVGlobalRegistry::getOpTypeFloat(uint32_t Width,200                                               MachineIRBuilder &MIRBuilder) {201  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {202    return MIRBuilder.buildInstr(SPIRV::OpTypeFloat)203        .addDef(createTypeVReg(MIRBuilder))204        .addImm(Width);205  });206}207 208SPIRVType *209SPIRVGlobalRegistry::getOpTypeFloat(uint32_t Width,210                                    MachineIRBuilder &MIRBuilder,211                                    SPIRV::FPEncoding::FPEncoding FPEncode) {212  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {213    return MIRBuilder.buildInstr(SPIRV::OpTypeFloat)214        .addDef(createTypeVReg(MIRBuilder))215        .addImm(Width)216        .addImm(FPEncode);217  });218}219 220SPIRVType *SPIRVGlobalRegistry::getOpTypeVoid(MachineIRBuilder &MIRBuilder) {221  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {222    return MIRBuilder.buildInstr(SPIRV::OpTypeVoid)223        .addDef(createTypeVReg(MIRBuilder));224  });225}226 227void SPIRVGlobalRegistry::invalidateMachineInstr(MachineInstr *MI) {228  // Other maps that may hold MachineInstr*:229  // - VRegToTypeMap: We cannot remove the definitions of `MI` from230  // VRegToTypeMap because some calls to invalidateMachineInstr are replacing MI231  // with another instruction defining the same register. We expect that if MI232  // is a type instruction, and it is still referenced in VRegToTypeMap, then233  // those registers are dead or the VRegToTypeMap is out-of-date. We do not234  // expect passes to ask for the SPIR-V type of a dead register. If the235  // VRegToTypeMap is out-of-date already, then there was an error before. We236  // cannot add an assert to verify this because the VRegToTypeMap can be237  // out-of-date.238  // - FunctionToInstr & FunctionToInstrRev: At this point, we should not be239  // deleting functions. No need to update.240  // - AliasInstMDMap: Would require a linear search, and the Intel Alias241  // instruction are not instructions instruction selection will be able to242  // remove.243 244  const SPIRVSubtarget &ST = MI->getMF()->getSubtarget<SPIRVSubtarget>();245  [[maybe_unused]] const SPIRVInstrInfo *TII = ST.getInstrInfo();246  assert(!TII->isAliasingInstr(*MI) &&247         "Cannot invalidate aliasing instructions.");248  assert(MI->getOpcode() != SPIRV::OpFunction &&249         "Cannot invalidate OpFunction.");250 251  if (MI->getOpcode() == SPIRV::OpFunctionCall) {252    if (const auto *F = dyn_cast<Function>(MI->getOperand(2).getGlobal())) {253      auto It = ForwardCalls.find(F);254      if (It != ForwardCalls.end()) {255        It->second.erase(MI);256        if (It->second.empty())257          ForwardCalls.erase(It);258      }259    }260  }261 262  const MachineFunction *MF = MI->getMF();263  auto It = LastInsertedTypeMap.find(MF);264  if (It != LastInsertedTypeMap.end() && It->second == MI)265    LastInsertedTypeMap.erase(MF);266  // remove from the duplicate tracker to avoid incorrect reuse267  erase(MI);268}269 270SPIRVType *SPIRVGlobalRegistry::createOpType(271    MachineIRBuilder &MIRBuilder,272    std::function<MachineInstr *(MachineIRBuilder &)> Op) {273  auto oldInsertPoint = MIRBuilder.getInsertPt();274  MachineBasicBlock *OldMBB = &MIRBuilder.getMBB();275  MachineBasicBlock *NewMBB = &*MIRBuilder.getMF().begin();276 277  auto LastInsertedType = LastInsertedTypeMap.find(CurMF);278  if (LastInsertedType != LastInsertedTypeMap.end()) {279    auto It = LastInsertedType->second->getIterator();280    // It might happen that this instruction was removed from the first MBB,281    // hence the Parent's check.282    MachineBasicBlock::iterator InsertAt;283    if (It->getParent() != NewMBB)284      InsertAt = oldInsertPoint->getParent() == NewMBB285                     ? oldInsertPoint286                     : getInsertPtValidEnd(NewMBB);287    else if (It->getNextNode())288      InsertAt = It->getNextNode()->getIterator();289    else290      InsertAt = getInsertPtValidEnd(NewMBB);291    MIRBuilder.setInsertPt(*NewMBB, InsertAt);292  } else {293    MIRBuilder.setInsertPt(*NewMBB, NewMBB->begin());294    auto Result = LastInsertedTypeMap.try_emplace(CurMF, nullptr);295    assert(Result.second);296    LastInsertedType = Result.first;297  }298 299  MachineInstr *Type = Op(MIRBuilder);300  // We expect all users of this function to insert definitions at the insertion301  // point set above that is always the first MBB.302  assert(Type->getParent() == NewMBB);303  LastInsertedType->second = Type;304 305  MIRBuilder.setInsertPt(*OldMBB, oldInsertPoint);306  return Type;307}308 309SPIRVType *SPIRVGlobalRegistry::getOpTypeVector(uint32_t NumElems,310                                                SPIRVType *ElemType,311                                                MachineIRBuilder &MIRBuilder) {312  auto EleOpc = ElemType->getOpcode();313  (void)EleOpc;314  assert((EleOpc == SPIRV::OpTypeInt || EleOpc == SPIRV::OpTypeFloat ||315          EleOpc == SPIRV::OpTypeBool) &&316         "Invalid vector element type");317 318  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {319    return MIRBuilder.buildInstr(SPIRV::OpTypeVector)320        .addDef(createTypeVReg(MIRBuilder))321        .addUse(getSPIRVTypeID(ElemType))322        .addImm(NumElems);323  });324}325 326Register SPIRVGlobalRegistry::getOrCreateConstFP(APFloat Val, MachineInstr &I,327                                                 SPIRVType *SpvType,328                                                 const SPIRVInstrInfo &TII,329                                                 bool ZeroAsNull) {330  LLVMContext &Ctx = CurMF->getFunction().getContext();331  auto *const CF = ConstantFP::get(Ctx, Val);332  const MachineInstr *MI = findMI(CF, CurMF);333  if (MI && (MI->getOpcode() == SPIRV::OpConstantNull ||334             MI->getOpcode() == SPIRV::OpConstantF))335    return MI->getOperand(0).getReg();336  return createConstFP(CF, I, SpvType, TII, ZeroAsNull);337}338 339Register SPIRVGlobalRegistry::createConstFP(const ConstantFP *CF,340                                            MachineInstr &I, SPIRVType *SpvType,341                                            const SPIRVInstrInfo &TII,342                                            bool ZeroAsNull) {343  unsigned BitWidth = getScalarOrVectorBitWidth(SpvType);344  LLT LLTy = LLT::scalar(BitWidth);345  Register Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);346  CurMF->getRegInfo().setRegClass(Res, &SPIRV::fIDRegClass);347  assignSPIRVTypeToVReg(SpvType, Res, *CurMF);348 349  MachineInstr *DepMI = const_cast<MachineInstr *>(SpvType);350  MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());351  SPIRVType *NewType =352      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {353        MachineInstrBuilder MIB;354        // In OpenCL OpConstantNull - Scalar floating point: +0.0 (all bits 0)355        if (CF->getValue().isPosZero() && ZeroAsNull) {356          MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)357                    .addDef(Res)358                    .addUse(getSPIRVTypeID(SpvType));359        } else {360          MIB = MIRBuilder.buildInstr(SPIRV::OpConstantF)361                    .addDef(Res)362                    .addUse(getSPIRVTypeID(SpvType));363          addNumImm(APInt(BitWidth,364                          CF->getValueAPF().bitcastToAPInt().getZExtValue()),365                    MIB);366        }367        const auto &ST = CurMF->getSubtarget();368        constrainSelectedInstRegOperands(*MIB, *ST.getInstrInfo(),369                                         *ST.getRegisterInfo(),370                                         *ST.getRegBankInfo());371        return MIB;372      });373  add(CF, NewType);374  return Res;375}376 377Register SPIRVGlobalRegistry::getOrCreateConstInt(uint64_t Val, MachineInstr &I,378                                                  SPIRVType *SpvType,379                                                  const SPIRVInstrInfo &TII,380                                                  bool ZeroAsNull) {381  const IntegerType *Ty = cast<IntegerType>(getTypeForSPIRVType(SpvType));382  auto *const CI = ConstantInt::get(const_cast<IntegerType *>(Ty), Val);383  const MachineInstr *MI = findMI(CI, CurMF);384  if (MI && (MI->getOpcode() == SPIRV::OpConstantNull ||385             MI->getOpcode() == SPIRV::OpConstantI))386    return MI->getOperand(0).getReg();387  return createConstInt(CI, I, SpvType, TII, ZeroAsNull);388}389 390Register SPIRVGlobalRegistry::createConstInt(const ConstantInt *CI,391                                             MachineInstr &I,392                                             SPIRVType *SpvType,393                                             const SPIRVInstrInfo &TII,394                                             bool ZeroAsNull) {395  unsigned BitWidth = getScalarOrVectorBitWidth(SpvType);396  LLT LLTy = LLT::scalar(BitWidth);397  Register Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);398  CurMF->getRegInfo().setRegClass(Res, &SPIRV::iIDRegClass);399  assignIntTypeToVReg(BitWidth, Res, I, TII);400 401  MachineInstr *DepMI = const_cast<MachineInstr *>(SpvType);402  MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());403  SPIRVType *NewType =404      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {405        MachineInstrBuilder MIB;406        if (BitWidth == 1) {407          MIB = MIRBuilder408                    .buildInstr(CI->isZero() ? SPIRV::OpConstantFalse409                                             : SPIRV::OpConstantTrue)410                    .addDef(Res)411                    .addUse(getSPIRVTypeID(SpvType));412        } else if (!CI->isZero() || !ZeroAsNull) {413          MIB = MIRBuilder.buildInstr(SPIRV::OpConstantI)414                    .addDef(Res)415                    .addUse(getSPIRVTypeID(SpvType));416          addNumImm(APInt(BitWidth, CI->getZExtValue()), MIB);417        } else {418          MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)419                    .addDef(Res)420                    .addUse(getSPIRVTypeID(SpvType));421        }422        const auto &ST = CurMF->getSubtarget();423        constrainSelectedInstRegOperands(*MIB, *ST.getInstrInfo(),424                                         *ST.getRegisterInfo(),425                                         *ST.getRegBankInfo());426        return MIB;427      });428  add(CI, NewType);429  return Res;430}431 432Register SPIRVGlobalRegistry::buildConstantInt(uint64_t Val,433                                               MachineIRBuilder &MIRBuilder,434                                               SPIRVType *SpvType, bool EmitIR,435                                               bool ZeroAsNull) {436  assert(SpvType);437  auto &MF = MIRBuilder.getMF();438  const IntegerType *Ty = cast<IntegerType>(getTypeForSPIRVType(SpvType));439  auto *const CI = ConstantInt::get(const_cast<IntegerType *>(Ty), Val);440  Register Res = find(CI, &MF);441  if (Res.isValid())442    return Res;443 444  unsigned BitWidth = getScalarOrVectorBitWidth(SpvType);445  LLT LLTy = LLT::scalar(BitWidth);446  MachineRegisterInfo &MRI = MF.getRegInfo();447  Res = MRI.createGenericVirtualRegister(LLTy);448  MRI.setRegClass(Res, &SPIRV::iIDRegClass);449  assignTypeToVReg(Ty, Res, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,450                   EmitIR);451 452  SPIRVType *NewType =453      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {454        if (EmitIR)455          return MIRBuilder.buildConstant(Res, *CI);456        Register SpvTypeReg = getSPIRVTypeID(SpvType);457        MachineInstrBuilder MIB;458        if (Val || !ZeroAsNull) {459          MIB = MIRBuilder.buildInstr(SPIRV::OpConstantI)460                    .addDef(Res)461                    .addUse(SpvTypeReg);462          addNumImm(APInt(BitWidth, Val), MIB);463        } else {464          MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)465                    .addDef(Res)466                    .addUse(SpvTypeReg);467        }468        const auto &Subtarget = CurMF->getSubtarget();469        constrainSelectedInstRegOperands(*MIB, *Subtarget.getInstrInfo(),470                                         *Subtarget.getRegisterInfo(),471                                         *Subtarget.getRegBankInfo());472        return MIB;473      });474  add(CI, NewType);475  return Res;476}477 478Register SPIRVGlobalRegistry::buildConstantFP(APFloat Val,479                                              MachineIRBuilder &MIRBuilder,480                                              SPIRVType *SpvType) {481  auto &MF = MIRBuilder.getMF();482  LLVMContext &Ctx = MF.getFunction().getContext();483  if (!SpvType)484    SpvType = getOrCreateSPIRVType(Type::getFloatTy(Ctx), MIRBuilder,485                                   SPIRV::AccessQualifier::ReadWrite, true);486  auto *const CF = ConstantFP::get(Ctx, Val);487  Register Res = find(CF, &MF);488  if (Res.isValid())489    return Res;490 491  LLT LLTy = LLT::scalar(getScalarOrVectorBitWidth(SpvType));492  Res = MF.getRegInfo().createGenericVirtualRegister(LLTy);493  MF.getRegInfo().setRegClass(Res, &SPIRV::fIDRegClass);494  assignSPIRVTypeToVReg(SpvType, Res, MF);495 496  SPIRVType *NewType =497      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {498        MachineInstrBuilder MIB;499        MIB = MIRBuilder.buildInstr(SPIRV::OpConstantF)500                  .addDef(Res)501                  .addUse(getSPIRVTypeID(SpvType));502        addNumImm(CF->getValueAPF().bitcastToAPInt(), MIB);503        return MIB;504      });505  add(CF, NewType);506  return Res;507}508 509Register SPIRVGlobalRegistry::getOrCreateBaseRegister(510    Constant *Val, MachineInstr &I, SPIRVType *SpvType,511    const SPIRVInstrInfo &TII, unsigned BitWidth, bool ZeroAsNull) {512  SPIRVType *Type = SpvType;513  if (SpvType->getOpcode() == SPIRV::OpTypeVector ||514      SpvType->getOpcode() == SPIRV::OpTypeArray) {515    auto EleTypeReg = SpvType->getOperand(1).getReg();516    Type = getSPIRVTypeForVReg(EleTypeReg);517  }518  if (Type->getOpcode() == SPIRV::OpTypeFloat) {519    SPIRVType *SpvBaseType = getOrCreateSPIRVFloatType(BitWidth, I, TII);520    return getOrCreateConstFP(cast<ConstantFP>(Val)->getValue(), I, SpvBaseType,521                              TII, ZeroAsNull);522  }523  assert(Type->getOpcode() == SPIRV::OpTypeInt);524  SPIRVType *SpvBaseType = getOrCreateSPIRVIntegerType(BitWidth, I, TII);525  return getOrCreateConstInt(Val->getUniqueInteger().getZExtValue(), I,526                             SpvBaseType, TII, ZeroAsNull);527}528 529Register SPIRVGlobalRegistry::getOrCreateCompositeOrNull(530    Constant *Val, MachineInstr &I, SPIRVType *SpvType,531    const SPIRVInstrInfo &TII, Constant *CA, unsigned BitWidth,532    unsigned ElemCnt, bool ZeroAsNull) {533  if (Register R = find(CA, CurMF); R.isValid())534    return R;535 536  bool IsNull = Val->isNullValue() && ZeroAsNull;537  Register ElemReg;538  if (!IsNull)539    ElemReg =540        getOrCreateBaseRegister(Val, I, SpvType, TII, BitWidth, ZeroAsNull);541 542  LLT LLTy = LLT::scalar(64);543  Register Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);544  CurMF->getRegInfo().setRegClass(Res, getRegClass(SpvType));545  assignSPIRVTypeToVReg(SpvType, Res, *CurMF);546 547  MachineInstr *DepMI = const_cast<MachineInstr *>(SpvType);548  MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());549  const MachineInstr *NewMI =550      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {551        MachineInstrBuilder MIB;552        if (!IsNull) {553          MIB = MIRBuilder.buildInstr(SPIRV::OpConstantComposite)554                    .addDef(Res)555                    .addUse(getSPIRVTypeID(SpvType));556          for (unsigned i = 0; i < ElemCnt; ++i)557            MIB.addUse(ElemReg);558        } else {559          MIB = MIRBuilder.buildInstr(SPIRV::OpConstantNull)560                    .addDef(Res)561                    .addUse(getSPIRVTypeID(SpvType));562        }563        const auto &Subtarget = CurMF->getSubtarget();564        constrainSelectedInstRegOperands(*MIB, *Subtarget.getInstrInfo(),565                                         *Subtarget.getRegisterInfo(),566                                         *Subtarget.getRegBankInfo());567        return MIB;568      });569  add(CA, NewMI);570  return Res;571}572 573Register SPIRVGlobalRegistry::getOrCreateConstVector(uint64_t Val,574                                                     MachineInstr &I,575                                                     SPIRVType *SpvType,576                                                     const SPIRVInstrInfo &TII,577                                                     bool ZeroAsNull) {578  const Type *LLVMTy = getTypeForSPIRVType(SpvType);579  assert(LLVMTy->isVectorTy());580  const FixedVectorType *LLVMVecTy = cast<FixedVectorType>(LLVMTy);581  Type *LLVMBaseTy = LLVMVecTy->getElementType();582  assert(LLVMBaseTy->isIntegerTy());583  auto *ConstVal = ConstantInt::get(LLVMBaseTy, Val);584  auto *ConstVec =585      ConstantVector::getSplat(LLVMVecTy->getElementCount(), ConstVal);586  unsigned BW = getScalarOrVectorBitWidth(SpvType);587  return getOrCreateCompositeOrNull(ConstVal, I, SpvType, TII, ConstVec, BW,588                                    SpvType->getOperand(2).getImm(),589                                    ZeroAsNull);590}591 592Register SPIRVGlobalRegistry::getOrCreateConstVector(APFloat Val,593                                                     MachineInstr &I,594                                                     SPIRVType *SpvType,595                                                     const SPIRVInstrInfo &TII,596                                                     bool ZeroAsNull) {597  const Type *LLVMTy = getTypeForSPIRVType(SpvType);598  assert(LLVMTy->isVectorTy());599  const FixedVectorType *LLVMVecTy = cast<FixedVectorType>(LLVMTy);600  Type *LLVMBaseTy = LLVMVecTy->getElementType();601  assert(LLVMBaseTy->isFloatingPointTy());602  auto *ConstVal = ConstantFP::get(LLVMBaseTy, Val);603  auto *ConstVec =604      ConstantVector::getSplat(LLVMVecTy->getElementCount(), ConstVal);605  unsigned BW = getScalarOrVectorBitWidth(SpvType);606  return getOrCreateCompositeOrNull(ConstVal, I, SpvType, TII, ConstVec, BW,607                                    SpvType->getOperand(2).getImm(),608                                    ZeroAsNull);609}610 611Register SPIRVGlobalRegistry::getOrCreateConstIntArray(612    uint64_t Val, size_t Num, MachineInstr &I, SPIRVType *SpvType,613    const SPIRVInstrInfo &TII) {614  const Type *LLVMTy = getTypeForSPIRVType(SpvType);615  assert(LLVMTy->isArrayTy());616  const ArrayType *LLVMArrTy = cast<ArrayType>(LLVMTy);617  Type *LLVMBaseTy = LLVMArrTy->getElementType();618  Constant *CI = ConstantInt::get(LLVMBaseTy, Val);619  SPIRVType *SpvBaseTy = getSPIRVTypeForVReg(SpvType->getOperand(1).getReg());620  unsigned BW = getScalarOrVectorBitWidth(SpvBaseTy);621  // The following is reasonably unique key that is better that [Val]. The naive622  // alternative would be something along the lines of:623  //   SmallVector<Constant *> NumCI(Num, CI);624  //   Constant *UniqueKey =625  //     ConstantArray::get(const_cast<ArrayType*>(LLVMArrTy), NumCI);626  // that would be a truly unique but dangerous key, because it could lead to627  // the creation of constants of arbitrary length (that is, the parameter of628  // memset) which were missing in the original module.629  Constant *UniqueKey = ConstantStruct::getAnon(630      {PoisonValue::get(const_cast<ArrayType *>(LLVMArrTy)),631       ConstantInt::get(LLVMBaseTy, Val), ConstantInt::get(LLVMBaseTy, Num)});632  return getOrCreateCompositeOrNull(CI, I, SpvType, TII, UniqueKey, BW,633                                    LLVMArrTy->getNumElements());634}635 636Register SPIRVGlobalRegistry::getOrCreateIntCompositeOrNull(637    uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVType *SpvType, bool EmitIR,638    Constant *CA, unsigned BitWidth, unsigned ElemCnt) {639  if (Register R = find(CA, CurMF); R.isValid())640    return R;641 642  Register ElemReg;643  if (Val || EmitIR) {644    SPIRVType *SpvBaseType = getOrCreateSPIRVIntegerType(BitWidth, MIRBuilder);645    ElemReg = buildConstantInt(Val, MIRBuilder, SpvBaseType, EmitIR);646  }647  LLT LLTy = EmitIR ? LLT::fixed_vector(ElemCnt, BitWidth) : LLT::scalar(64);648  Register Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);649  CurMF->getRegInfo().setRegClass(Res, &SPIRV::iIDRegClass);650  assignSPIRVTypeToVReg(SpvType, Res, *CurMF);651 652  const MachineInstr *NewMI =653      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {654        if (EmitIR)655          return MIRBuilder.buildSplatBuildVector(Res, ElemReg);656 657        if (Val) {658          auto MIB = MIRBuilder.buildInstr(SPIRV::OpConstantComposite)659                         .addDef(Res)660                         .addUse(getSPIRVTypeID(SpvType));661          for (unsigned i = 0; i < ElemCnt; ++i)662            MIB.addUse(ElemReg);663          return MIB;664        }665 666        return MIRBuilder.buildInstr(SPIRV::OpConstantNull)667            .addDef(Res)668            .addUse(getSPIRVTypeID(SpvType));669      });670  add(CA, NewMI);671  return Res;672}673 674Register675SPIRVGlobalRegistry::getOrCreateConsIntVector(uint64_t Val,676                                              MachineIRBuilder &MIRBuilder,677                                              SPIRVType *SpvType, bool EmitIR) {678  const Type *LLVMTy = getTypeForSPIRVType(SpvType);679  assert(LLVMTy->isVectorTy());680  const FixedVectorType *LLVMVecTy = cast<FixedVectorType>(LLVMTy);681  Type *LLVMBaseTy = LLVMVecTy->getElementType();682  const auto ConstInt = ConstantInt::get(LLVMBaseTy, Val);683  auto ConstVec =684      ConstantVector::getSplat(LLVMVecTy->getElementCount(), ConstInt);685  unsigned BW = getScalarOrVectorBitWidth(SpvType);686  return getOrCreateIntCompositeOrNull(Val, MIRBuilder, SpvType, EmitIR,687                                       ConstVec, BW,688                                       SpvType->getOperand(2).getImm());689}690 691Register692SPIRVGlobalRegistry::getOrCreateConstNullPtr(MachineIRBuilder &MIRBuilder,693                                             SPIRVType *SpvType) {694  const Type *Ty = getTypeForSPIRVType(SpvType);695  unsigned AddressSpace = typeToAddressSpace(Ty);696  Type *ElemTy = ::getPointeeType(Ty);697  assert(ElemTy);698  const Constant *CP = ConstantTargetNone::get(699      dyn_cast<TargetExtType>(getTypedPointerWrapper(ElemTy, AddressSpace)));700  Register Res = find(CP, CurMF);701  if (Res.isValid())702    return Res;703 704  LLT LLTy = LLT::pointer(AddressSpace, PointerSize);705  Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);706  CurMF->getRegInfo().setRegClass(Res, &SPIRV::pIDRegClass);707  assignSPIRVTypeToVReg(SpvType, Res, *CurMF);708 709  const MachineInstr *NewMI =710      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {711        return MIRBuilder.buildInstr(SPIRV::OpConstantNull)712            .addDef(Res)713            .addUse(getSPIRVTypeID(SpvType));714      });715  add(CP, NewMI);716  return Res;717}718 719Register720SPIRVGlobalRegistry::buildConstantSampler(Register ResReg, unsigned AddrMode,721                                          unsigned Param, unsigned FilerMode,722                                          MachineIRBuilder &MIRBuilder) {723  auto Sampler =724      ResReg.isValid()725          ? ResReg726          : MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);727  SPIRVType *TypeSampler = getOrCreateOpTypeSampler(MIRBuilder);728  Register TypeSamplerReg = getSPIRVTypeID(TypeSampler);729  // We cannot use createOpType() logic here, because of the730  // GlobalISel/IRTranslator.cpp check for a tail call that expects that731  // MIRBuilder.getInsertPt() has a previous instruction. If this constant is732  // inserted as a result of "__translate_sampler_initializer()" this would733  // break this IRTranslator assumption.734  MIRBuilder.buildInstr(SPIRV::OpConstantSampler)735      .addDef(Sampler)736      .addUse(TypeSamplerReg)737      .addImm(AddrMode)738      .addImm(Param)739      .addImm(FilerMode);740  return Sampler;741}742 743Register SPIRVGlobalRegistry::buildGlobalVariable(744    Register ResVReg, SPIRVType *BaseType, StringRef Name,745    const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage,746    const MachineInstr *Init, bool IsConst,747    const std::optional<SPIRV::LinkageType::LinkageType> &LinkageType,748    MachineIRBuilder &MIRBuilder, bool IsInstSelector) {749  const GlobalVariable *GVar = nullptr;750  if (GV) {751    GVar = cast<const GlobalVariable>(GV);752  } else {753    // If GV is not passed explicitly, use the name to find or construct754    // the global variable.755    Module *M = MIRBuilder.getMF().getFunction().getParent();756    GVar = M->getGlobalVariable(Name);757    if (GVar == nullptr) {758      const Type *Ty = getTypeForSPIRVType(BaseType); // TODO: check type.759      // Module takes ownership of the global var.760      GVar = new GlobalVariable(*M, const_cast<Type *>(Ty), false,761                                GlobalValue::ExternalLinkage, nullptr,762                                Twine(Name));763    }764    GV = GVar;765  }766 767  const MachineFunction *MF = &MIRBuilder.getMF();768  Register Reg = find(GVar, MF);769  if (Reg.isValid()) {770    if (Reg != ResVReg)771      MIRBuilder.buildCopy(ResVReg, Reg);772    return ResVReg;773  }774 775  auto MIB = MIRBuilder.buildInstr(SPIRV::OpVariable)776                 .addDef(ResVReg)777                 .addUse(getSPIRVTypeID(BaseType))778                 .addImm(static_cast<uint32_t>(Storage));779  if (Init)780    MIB.addUse(Init->getOperand(0).getReg());781  // ISel may introduce a new register on this step, so we need to add it to782  // DT and correct its type avoiding fails on the next stage.783  if (IsInstSelector) {784    const auto &Subtarget = CurMF->getSubtarget();785    constrainSelectedInstRegOperands(*MIB, *Subtarget.getInstrInfo(),786                                     *Subtarget.getRegisterInfo(),787                                     *Subtarget.getRegBankInfo());788  }789  add(GVar, MIB);790 791  Reg = MIB->getOperand(0).getReg();792  addGlobalObject(GVar, MF, Reg);793 794  // Set to Reg the same type as ResVReg has.795  auto MRI = MIRBuilder.getMRI();796  if (Reg != ResVReg) {797    LLT RegLLTy =798        LLT::pointer(MRI->getType(ResVReg).getAddressSpace(), getPointerSize());799    MRI->setType(Reg, RegLLTy);800    assignSPIRVTypeToVReg(BaseType, Reg, MIRBuilder.getMF());801  } else {802    // Our knowledge about the type may be updated.803    // If that's the case, we need to update a type804    // associated with the register.805    SPIRVType *DefType = getSPIRVTypeForVReg(ResVReg);806    if (!DefType || DefType != BaseType)807      assignSPIRVTypeToVReg(BaseType, Reg, MIRBuilder.getMF());808  }809 810  // If it's a global variable with name, output OpName for it.811  if (GVar && GVar->hasName())812    buildOpName(Reg, GVar->getName(), MIRBuilder);813 814  // Output decorations for the GV.815  // TODO: maybe move to GenerateDecorations pass.816  const SPIRVSubtarget &ST =817      cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget());818  if (IsConst && !ST.isShader())819    buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Constant, {});820 821  if (GVar && GVar->getAlign().valueOrOne().value() != 1 && !ST.isShader()) {822    unsigned Alignment = (unsigned)GVar->getAlign().valueOrOne().value();823    buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Alignment, {Alignment});824  }825 826  if (LinkageType)827    buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::LinkageAttributes,828                    {static_cast<uint32_t>(*LinkageType)}, Name);829 830  SPIRV::BuiltIn::BuiltIn BuiltInId;831  if (getSpirvBuiltInIdByName(Name, BuiltInId))832    buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::BuiltIn,833                    {static_cast<uint32_t>(BuiltInId)});834 835  // If it's a global variable with "spirv.Decorations" metadata node836  // recognize it as a SPIR-V friendly LLVM IR and parse "spirv.Decorations"837  // arguments.838  MDNode *GVarMD = nullptr;839  if (GVar && (GVarMD = GVar->getMetadata("spirv.Decorations")) != nullptr)840    buildOpSpirvDecorations(Reg, MIRBuilder, GVarMD, ST);841 842  return Reg;843}844 845// Returns a name based on the Type. Notes that this does not look at846// decorations, and will return the same string for two types that are the same847// except for decorations.848Register SPIRVGlobalRegistry::getOrCreateGlobalVariableWithBinding(849    const SPIRVType *VarType, uint32_t Set, uint32_t Binding, StringRef Name,850    MachineIRBuilder &MIRBuilder) {851  Register VarReg =852      MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass);853 854  buildGlobalVariable(VarReg, VarType, Name, nullptr,855                      getPointerStorageClass(VarType), nullptr, false,856                      std::nullopt, MIRBuilder, false);857 858  buildOpDecorate(VarReg, MIRBuilder, SPIRV::Decoration::DescriptorSet, {Set});859  buildOpDecorate(VarReg, MIRBuilder, SPIRV::Decoration::Binding, {Binding});860  return VarReg;861}862 863// TODO: Double check the calls to getOpTypeArray to make sure that `ElemType`864// is explicitly laid out when required.865SPIRVType *SPIRVGlobalRegistry::getOpTypeArray(uint32_t NumElems,866                                               SPIRVType *ElemType,867                                               MachineIRBuilder &MIRBuilder,868                                               bool ExplicitLayoutRequired,869                                               bool EmitIR) {870  assert((ElemType->getOpcode() != SPIRV::OpTypeVoid) &&871         "Invalid array element type");872  SPIRVType *SpvTypeInt32 = getOrCreateSPIRVIntegerType(32, MIRBuilder);873  SPIRVType *ArrayType = nullptr;874  const SPIRVSubtarget &ST =875      cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget());876  if (NumElems != 0) {877    Register NumElementsVReg =878        buildConstantInt(NumElems, MIRBuilder, SpvTypeInt32, EmitIR);879    ArrayType = createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {880      return MIRBuilder.buildInstr(SPIRV::OpTypeArray)881          .addDef(createTypeVReg(MIRBuilder))882          .addUse(getSPIRVTypeID(ElemType))883          .addUse(NumElementsVReg);884    });885  } else {886    assert(ST.isShader() && "Runtime arrays are not allowed in non-shader "887                            "SPIR-V modules.");888    if (!ST.isShader())889      return nullptr;890    ArrayType = createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {891      return MIRBuilder.buildInstr(SPIRV::OpTypeRuntimeArray)892          .addDef(createTypeVReg(MIRBuilder))893          .addUse(getSPIRVTypeID(ElemType));894    });895  }896 897  if (ExplicitLayoutRequired && !isResourceType(ElemType)) {898    Type *ET = const_cast<Type *>(getTypeForSPIRVType(ElemType));899    addArrayStrideDecorations(ArrayType->defs().begin()->getReg(), ET,900                              MIRBuilder);901  }902 903  return ArrayType;904}905 906SPIRVType *SPIRVGlobalRegistry::getOpTypeOpaque(const StructType *Ty,907                                                MachineIRBuilder &MIRBuilder) {908  assert(Ty->hasName());909  const StringRef Name = Ty->hasName() ? Ty->getName() : "";910  Register ResVReg = createTypeVReg(MIRBuilder);911  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {912    auto MIB = MIRBuilder.buildInstr(SPIRV::OpTypeOpaque).addDef(ResVReg);913    addStringImm(Name, MIB);914    buildOpName(ResVReg, Name, MIRBuilder);915    return MIB;916  });917}918 919SPIRVType *SPIRVGlobalRegistry::getOpTypeStruct(920    const StructType *Ty, MachineIRBuilder &MIRBuilder,921    SPIRV::AccessQualifier::AccessQualifier AccQual,922    StructOffsetDecorator Decorator, bool EmitIR) {923  Type *OriginalElementType = nullptr;924  uint64_t TotalSize = 0;925  if (matchPeeledArrayPattern(Ty, OriginalElementType, TotalSize)) {926    SPIRVType *ElementSPIRVType = findSPIRVType(927        OriginalElementType, MIRBuilder, AccQual,928        /* ExplicitLayoutRequired= */ Decorator != nullptr, EmitIR);929    return getOpTypeArray(TotalSize, ElementSPIRVType, MIRBuilder,930                          /*ExplicitLayoutRequired=*/Decorator != nullptr,931                          EmitIR);932  }933 934  const SPIRVSubtarget &ST =935      cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget());936  SmallVector<Register, 4> FieldTypes;937  constexpr unsigned MaxWordCount = UINT16_MAX;938  const size_t NumElements = Ty->getNumElements();939 940  size_t MaxNumElements = MaxWordCount - 2;941  size_t SPIRVStructNumElements = NumElements;942  if (NumElements > MaxNumElements) {943    // Do adjustments for continued instructions.944    SPIRVStructNumElements = MaxNumElements;945    MaxNumElements = MaxWordCount - 1;946  }947 948  for (const auto &Elem : Ty->elements()) {949    SPIRVType *ElemTy = findSPIRVType(950        toTypedPointer(Elem), MIRBuilder, AccQual,951        /* ExplicitLayoutRequired= */ Decorator != nullptr, EmitIR);952    assert(ElemTy && ElemTy->getOpcode() != SPIRV::OpTypeVoid &&953           "Invalid struct element type");954    FieldTypes.push_back(getSPIRVTypeID(ElemTy));955  }956  Register ResVReg = createTypeVReg(MIRBuilder);957  if (Ty->hasName())958    buildOpName(ResVReg, Ty->getName(), MIRBuilder);959  if (Ty->isPacked() && !ST.isShader())960    buildOpDecorate(ResVReg, MIRBuilder, SPIRV::Decoration::CPacked, {});961 962  SPIRVType *SPVType =963      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {964        auto MIBStruct =965            MIRBuilder.buildInstr(SPIRV::OpTypeStruct).addDef(ResVReg);966        for (size_t I = 0; I < SPIRVStructNumElements; ++I)967          MIBStruct.addUse(FieldTypes[I]);968        for (size_t I = SPIRVStructNumElements; I < NumElements;969             I += MaxNumElements) {970          auto MIBCont =971              MIRBuilder.buildInstr(SPIRV::OpTypeStructContinuedINTEL);972          for (size_t J = I; J < std::min(I + MaxNumElements, NumElements); ++J)973            MIBCont.addUse(FieldTypes[I]);974        }975        return MIBStruct;976      });977 978  if (Decorator)979    Decorator(SPVType->defs().begin()->getReg());980 981  return SPVType;982}983 984SPIRVType *SPIRVGlobalRegistry::getOrCreateSpecialType(985    const Type *Ty, MachineIRBuilder &MIRBuilder,986    SPIRV::AccessQualifier::AccessQualifier AccQual) {987  assert(isSpecialOpaqueType(Ty) && "Not a special opaque builtin type");988  return SPIRV::lowerBuiltinType(Ty, AccQual, MIRBuilder, this);989}990 991SPIRVType *SPIRVGlobalRegistry::getOpTypePointer(992    SPIRV::StorageClass::StorageClass SC, SPIRVType *ElemType,993    MachineIRBuilder &MIRBuilder, Register Reg) {994  if (!Reg.isValid())995    Reg = createTypeVReg(MIRBuilder);996 997  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {998    return MIRBuilder.buildInstr(SPIRV::OpTypePointer)999        .addDef(Reg)1000        .addImm(static_cast<uint32_t>(SC))1001        .addUse(getSPIRVTypeID(ElemType));1002  });1003}1004 1005SPIRVType *SPIRVGlobalRegistry::getOpTypeForwardPointer(1006    SPIRV::StorageClass::StorageClass SC, MachineIRBuilder &MIRBuilder) {1007  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1008    return MIRBuilder.buildInstr(SPIRV::OpTypeForwardPointer)1009        .addUse(createTypeVReg(MIRBuilder))1010        .addImm(static_cast<uint32_t>(SC));1011  });1012}1013 1014SPIRVType *SPIRVGlobalRegistry::getOpTypeFunction(1015    const FunctionType *Ty, SPIRVType *RetType,1016    const SmallVectorImpl<SPIRVType *> &ArgTypes,1017    MachineIRBuilder &MIRBuilder) {1018  if (Ty->isVarArg()) {1019    Function &Fn = MIRBuilder.getMF().getFunction();1020    Ty->getContext().diagnose(DiagnosticInfoUnsupported(1021        Fn, "SPIR-V does not support variadic functions",1022        MIRBuilder.getDebugLoc()));1023  }1024  return createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1025    auto MIB = MIRBuilder.buildInstr(SPIRV::OpTypeFunction)1026                   .addDef(createTypeVReg(MIRBuilder))1027                   .addUse(getSPIRVTypeID(RetType));1028    for (const SPIRVType *ArgType : ArgTypes)1029      MIB.addUse(getSPIRVTypeID(ArgType));1030    return MIB;1031  });1032}1033 1034SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeFunctionWithArgs(1035    const Type *Ty, SPIRVType *RetType,1036    const SmallVectorImpl<SPIRVType *> &ArgTypes,1037    MachineIRBuilder &MIRBuilder) {1038  if (const MachineInstr *MI = findMI(Ty, false, &MIRBuilder.getMF()))1039    return MI;1040  const MachineInstr *NewMI =1041      getOpTypeFunction(cast<FunctionType>(Ty), RetType, ArgTypes, MIRBuilder);1042  add(Ty, false, NewMI);1043  return finishCreatingSPIRVType(Ty, NewMI);1044}1045 1046SPIRVType *SPIRVGlobalRegistry::findSPIRVType(1047    const Type *Ty, MachineIRBuilder &MIRBuilder,1048    SPIRV::AccessQualifier::AccessQualifier AccQual,1049    bool ExplicitLayoutRequired, bool EmitIR) {1050  Ty = adjustIntTypeByWidth(Ty);1051  // TODO: findMI needs to know if a layout is required.1052  if (const MachineInstr *MI =1053          findMI(Ty, ExplicitLayoutRequired, &MIRBuilder.getMF()))1054    return MI;1055  if (auto It = ForwardPointerTypes.find(Ty); It != ForwardPointerTypes.end())1056    return It->second;1057  return restOfCreateSPIRVType(Ty, MIRBuilder, AccQual, ExplicitLayoutRequired,1058                               EmitIR);1059}1060 1061Register SPIRVGlobalRegistry::getSPIRVTypeID(const SPIRVType *SpirvType) const {1062  assert(SpirvType && "Attempting to get type id for nullptr type.");1063  if (SpirvType->getOpcode() == SPIRV::OpTypeForwardPointer ||1064      SpirvType->getOpcode() == SPIRV::OpTypeStructContinuedINTEL)1065    return SpirvType->uses().begin()->getReg();1066  return SpirvType->defs().begin()->getReg();1067}1068 1069// We need to use a new LLVM integer type if there is a mismatch between1070// number of bits in LLVM and SPIRV integer types to let DuplicateTracker1071// ensure uniqueness of a SPIRV type by the corresponding LLVM type. Without1072// such an adjustment SPIRVGlobalRegistry::getOpTypeInt() could create the1073// same "OpTypeInt 8" type for a series of LLVM integer types with number of1074// bits less than 8. This would lead to duplicate type definitions1075// eventually due to the method that DuplicateTracker utilizes to reason1076// about uniqueness of type records.1077const Type *SPIRVGlobalRegistry::adjustIntTypeByWidth(const Type *Ty) const {1078  if (auto IType = dyn_cast<IntegerType>(Ty)) {1079    unsigned SrcBitWidth = IType->getBitWidth();1080    if (SrcBitWidth > 1) {1081      unsigned BitWidth = adjustOpTypeIntWidth(SrcBitWidth);1082      // Maybe change source LLVM type to keep DuplicateTracker consistent.1083      if (SrcBitWidth != BitWidth)1084        Ty = IntegerType::get(Ty->getContext(), BitWidth);1085    }1086  }1087  return Ty;1088}1089 1090SPIRVType *SPIRVGlobalRegistry::createSPIRVType(1091    const Type *Ty, MachineIRBuilder &MIRBuilder,1092    SPIRV::AccessQualifier::AccessQualifier AccQual,1093    bool ExplicitLayoutRequired, bool EmitIR) {1094  if (isSpecialOpaqueType(Ty))1095    return getOrCreateSpecialType(Ty, MIRBuilder, AccQual);1096 1097  if (const MachineInstr *MI =1098          findMI(Ty, ExplicitLayoutRequired, &MIRBuilder.getMF()))1099    return MI;1100 1101  if (auto IType = dyn_cast<IntegerType>(Ty)) {1102    const unsigned Width = IType->getBitWidth();1103    return Width == 1 ? getOpTypeBool(MIRBuilder)1104                      : getOpTypeInt(Width, MIRBuilder, false);1105  }1106  if (Ty->isFloatingPointTy()) {1107    if (Ty->isBFloatTy()) {1108      return getOpTypeFloat(Ty->getPrimitiveSizeInBits(), MIRBuilder,1109                            SPIRV::FPEncoding::BFloat16KHR);1110    } else {1111      return getOpTypeFloat(Ty->getPrimitiveSizeInBits(), MIRBuilder);1112    }1113  }1114  if (Ty->isVoidTy())1115    return getOpTypeVoid(MIRBuilder);1116  if (Ty->isVectorTy()) {1117    SPIRVType *El =1118        findSPIRVType(cast<FixedVectorType>(Ty)->getElementType(), MIRBuilder,1119                      AccQual, ExplicitLayoutRequired, EmitIR);1120    return getOpTypeVector(cast<FixedVectorType>(Ty)->getNumElements(), El,1121                           MIRBuilder);1122  }1123  if (Ty->isArrayTy()) {1124    SPIRVType *El = findSPIRVType(Ty->getArrayElementType(), MIRBuilder,1125                                  AccQual, ExplicitLayoutRequired, EmitIR);1126    return getOpTypeArray(Ty->getArrayNumElements(), El, MIRBuilder,1127                          ExplicitLayoutRequired, EmitIR);1128  }1129  if (auto SType = dyn_cast<StructType>(Ty)) {1130    if (SType->isOpaque())1131      return getOpTypeOpaque(SType, MIRBuilder);1132 1133    StructOffsetDecorator Decorator = nullptr;1134    if (ExplicitLayoutRequired) {1135      Decorator = [&MIRBuilder, SType, this](Register Reg) {1136        addStructOffsetDecorations(Reg, const_cast<StructType *>(SType),1137                                   MIRBuilder);1138      };1139    }1140    return getOpTypeStruct(SType, MIRBuilder, AccQual, std::move(Decorator),1141                           EmitIR);1142  }1143  if (auto FType = dyn_cast<FunctionType>(Ty)) {1144    SPIRVType *RetTy = findSPIRVType(FType->getReturnType(), MIRBuilder,1145                                     AccQual, ExplicitLayoutRequired, EmitIR);1146    SmallVector<SPIRVType *, 4> ParamTypes;1147    for (const auto &ParamTy : FType->params())1148      ParamTypes.push_back(findSPIRVType(ParamTy, MIRBuilder, AccQual,1149                                         ExplicitLayoutRequired, EmitIR));1150    return getOpTypeFunction(FType, RetTy, ParamTypes, MIRBuilder);1151  }1152 1153  unsigned AddrSpace = typeToAddressSpace(Ty);1154  SPIRVType *SpvElementType = nullptr;1155  if (Type *ElemTy = ::getPointeeType(Ty))1156    SpvElementType = getOrCreateSPIRVType(ElemTy, MIRBuilder, AccQual, EmitIR);1157  else1158    SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder);1159 1160  // Get access to information about available extensions1161  const SPIRVSubtarget *ST =1162      static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());1163  auto SC = addressSpaceToStorageClass(AddrSpace, *ST);1164 1165  Type *ElemTy = ::getPointeeType(Ty);1166  if (!ElemTy) {1167    ElemTy = Type::getInt8Ty(MIRBuilder.getContext());1168  }1169 1170  // If we have forward pointer associated with this type, use its register1171  // operand to create OpTypePointer.1172  if (auto It = ForwardPointerTypes.find(Ty); It != ForwardPointerTypes.end()) {1173    Register Reg = getSPIRVTypeID(It->second);1174    // TODO: what does getOpTypePointer do?1175    return getOpTypePointer(SC, SpvElementType, MIRBuilder, Reg);1176  }1177 1178  return getOrCreateSPIRVPointerType(ElemTy, MIRBuilder, SC);1179}1180 1181SPIRVType *SPIRVGlobalRegistry::restOfCreateSPIRVType(1182    const Type *Ty, MachineIRBuilder &MIRBuilder,1183    SPIRV::AccessQualifier::AccessQualifier AccessQual,1184    bool ExplicitLayoutRequired, bool EmitIR) {1185  // TODO: Could this create a problem if one requires an explicit layout, and1186  // the next time it does not?1187  if (TypesInProcessing.count(Ty) && !isPointerTyOrWrapper(Ty))1188    return nullptr;1189  TypesInProcessing.insert(Ty);1190  SPIRVType *SpirvType = createSPIRVType(Ty, MIRBuilder, AccessQual,1191                                         ExplicitLayoutRequired, EmitIR);1192  TypesInProcessing.erase(Ty);1193  VRegToTypeMap[&MIRBuilder.getMF()][getSPIRVTypeID(SpirvType)] = SpirvType;1194 1195  // TODO: We could end up with two SPIR-V types pointing to the same llvm type.1196  // Is that a problem?1197  SPIRVToLLVMType[SpirvType] = unifyPtrType(Ty);1198 1199  if (SpirvType->getOpcode() == SPIRV::OpTypeForwardPointer ||1200      findMI(Ty, false, &MIRBuilder.getMF()) || isSpecialOpaqueType(Ty))1201    return SpirvType;1202 1203  if (auto *ExtTy = dyn_cast<TargetExtType>(Ty);1204      ExtTy && isTypedPointerWrapper(ExtTy))1205    add(ExtTy->getTypeParameter(0), ExtTy->getIntParameter(0), SpirvType);1206  else if (!isPointerTy(Ty))1207    add(Ty, ExplicitLayoutRequired, SpirvType);1208  else if (isTypedPointerTy(Ty))1209    add(cast<TypedPointerType>(Ty)->getElementType(),1210        getPointerAddressSpace(Ty), SpirvType);1211  else1212    add(Type::getInt8Ty(MIRBuilder.getMF().getFunction().getContext()),1213        getPointerAddressSpace(Ty), SpirvType);1214  return SpirvType;1215}1216 1217SPIRVType *1218SPIRVGlobalRegistry::getSPIRVTypeForVReg(Register VReg,1219                                         const MachineFunction *MF) const {1220  auto t = VRegToTypeMap.find(MF ? MF : CurMF);1221  if (t != VRegToTypeMap.end()) {1222    auto tt = t->second.find(VReg);1223    if (tt != t->second.end())1224      return tt->second;1225  }1226  return nullptr;1227}1228 1229SPIRVType *SPIRVGlobalRegistry::getResultType(Register VReg,1230                                              MachineFunction *MF) {1231  if (!MF)1232    MF = CurMF;1233  MachineInstr *Instr = getVRegDef(MF->getRegInfo(), VReg);1234  return getSPIRVTypeForVReg(Instr->getOperand(1).getReg(), MF);1235}1236 1237SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVType(1238    const Type *Ty, MachineIRBuilder &MIRBuilder,1239    SPIRV::AccessQualifier::AccessQualifier AccessQual,1240    bool ExplicitLayoutRequired, bool EmitIR) {1241  const MachineFunction *MF = &MIRBuilder.getMF();1242  Register Reg;1243  if (auto *ExtTy = dyn_cast<TargetExtType>(Ty);1244      ExtTy && isTypedPointerWrapper(ExtTy))1245    Reg = find(ExtTy->getTypeParameter(0), ExtTy->getIntParameter(0), MF);1246  else if (!isPointerTy(Ty))1247    Reg = find(Ty = adjustIntTypeByWidth(Ty), ExplicitLayoutRequired, MF);1248  else if (isTypedPointerTy(Ty))1249    Reg = find(cast<TypedPointerType>(Ty)->getElementType(),1250               getPointerAddressSpace(Ty), MF);1251  else1252    Reg = find(Type::getInt8Ty(MIRBuilder.getMF().getFunction().getContext()),1253               getPointerAddressSpace(Ty), MF);1254  if (Reg.isValid() && !isSpecialOpaqueType(Ty))1255    return getSPIRVTypeForVReg(Reg);1256 1257  TypesInProcessing.clear();1258  SPIRVType *STy = restOfCreateSPIRVType(Ty, MIRBuilder, AccessQual,1259                                         ExplicitLayoutRequired, EmitIR);1260  // Create normal pointer types for the corresponding OpTypeForwardPointers.1261  for (auto &CU : ForwardPointerTypes) {1262    // Pointer type themselves do not require an explicit layout. The types1263    // they pointer to might, but that is taken care of when creating the type.1264    bool PtrNeedsLayout = false;1265    const Type *Ty2 = CU.first;1266    SPIRVType *STy2 = CU.second;1267    if ((Reg = find(Ty2, PtrNeedsLayout, MF)).isValid())1268      STy2 = getSPIRVTypeForVReg(Reg);1269    else1270      STy2 = restOfCreateSPIRVType(Ty2, MIRBuilder, AccessQual, PtrNeedsLayout,1271                                   EmitIR);1272    if (Ty == Ty2)1273      STy = STy2;1274  }1275  ForwardPointerTypes.clear();1276  return STy;1277}1278 1279bool SPIRVGlobalRegistry::isScalarOfType(Register VReg,1280                                         unsigned TypeOpcode) const {1281  SPIRVType *Type = getSPIRVTypeForVReg(VReg);1282  assert(Type && "isScalarOfType VReg has no type assigned");1283  return Type->getOpcode() == TypeOpcode;1284}1285 1286bool SPIRVGlobalRegistry::isScalarOrVectorOfType(Register VReg,1287                                                 unsigned TypeOpcode) const {1288  SPIRVType *Type = getSPIRVTypeForVReg(VReg);1289  assert(Type && "isScalarOrVectorOfType VReg has no type assigned");1290  if (Type->getOpcode() == TypeOpcode)1291    return true;1292  if (Type->getOpcode() == SPIRV::OpTypeVector) {1293    Register ScalarTypeVReg = Type->getOperand(1).getReg();1294    SPIRVType *ScalarType = getSPIRVTypeForVReg(ScalarTypeVReg);1295    return ScalarType->getOpcode() == TypeOpcode;1296  }1297  return false;1298}1299 1300bool SPIRVGlobalRegistry::isResourceType(SPIRVType *Type) const {1301  switch (Type->getOpcode()) {1302  case SPIRV::OpTypeImage:1303  case SPIRV::OpTypeSampler:1304  case SPIRV::OpTypeSampledImage:1305    return true;1306  case SPIRV::OpTypeStruct:1307    return hasBlockDecoration(Type);1308  default:1309    return false;1310  }1311  return false;1312}1313unsigned1314SPIRVGlobalRegistry::getScalarOrVectorComponentCount(Register VReg) const {1315  return getScalarOrVectorComponentCount(getSPIRVTypeForVReg(VReg));1316}1317 1318unsigned1319SPIRVGlobalRegistry::getScalarOrVectorComponentCount(SPIRVType *Type) const {1320  if (!Type)1321    return 0;1322  return Type->getOpcode() == SPIRV::OpTypeVector1323             ? static_cast<unsigned>(Type->getOperand(2).getImm())1324             : 1;1325}1326 1327SPIRVType *1328SPIRVGlobalRegistry::getScalarOrVectorComponentType(Register VReg) const {1329  return getScalarOrVectorComponentType(getSPIRVTypeForVReg(VReg));1330}1331 1332SPIRVType *1333SPIRVGlobalRegistry::getScalarOrVectorComponentType(SPIRVType *Type) const {1334  if (!Type)1335    return nullptr;1336  Register ScalarReg = Type->getOpcode() == SPIRV::OpTypeVector1337                           ? Type->getOperand(1).getReg()1338                           : Type->getOperand(0).getReg();1339  SPIRVType *ScalarType = getSPIRVTypeForVReg(ScalarReg);1340  assert(isScalarOrVectorOfType(Type->getOperand(0).getReg(),1341                                ScalarType->getOpcode()));1342  return ScalarType;1343}1344 1345unsigned1346SPIRVGlobalRegistry::getScalarOrVectorBitWidth(const SPIRVType *Type) const {1347  assert(Type && "Invalid Type pointer");1348  if (Type->getOpcode() == SPIRV::OpTypeVector) {1349    auto EleTypeReg = Type->getOperand(1).getReg();1350    Type = getSPIRVTypeForVReg(EleTypeReg);1351  }1352  if (Type->getOpcode() == SPIRV::OpTypeInt ||1353      Type->getOpcode() == SPIRV::OpTypeFloat)1354    return Type->getOperand(1).getImm();1355  if (Type->getOpcode() == SPIRV::OpTypeBool)1356    return 1;1357  llvm_unreachable("Attempting to get bit width of non-integer/float type.");1358}1359 1360unsigned SPIRVGlobalRegistry::getNumScalarOrVectorTotalBitWidth(1361    const SPIRVType *Type) const {1362  assert(Type && "Invalid Type pointer");1363  unsigned NumElements = 1;1364  if (Type->getOpcode() == SPIRV::OpTypeVector) {1365    NumElements = static_cast<unsigned>(Type->getOperand(2).getImm());1366    Type = getSPIRVTypeForVReg(Type->getOperand(1).getReg());1367  }1368  return Type->getOpcode() == SPIRV::OpTypeInt ||1369                 Type->getOpcode() == SPIRV::OpTypeFloat1370             ? NumElements * Type->getOperand(1).getImm()1371             : 0;1372}1373 1374const SPIRVType *SPIRVGlobalRegistry::retrieveScalarOrVectorIntType(1375    const SPIRVType *Type) const {1376  if (Type && Type->getOpcode() == SPIRV::OpTypeVector)1377    Type = getSPIRVTypeForVReg(Type->getOperand(1).getReg());1378  return Type && Type->getOpcode() == SPIRV::OpTypeInt ? Type : nullptr;1379}1380 1381bool SPIRVGlobalRegistry::isScalarOrVectorSigned(const SPIRVType *Type) const {1382  const SPIRVType *IntType = retrieveScalarOrVectorIntType(Type);1383  return IntType && IntType->getOperand(2).getImm() != 0;1384}1385 1386SPIRVType *SPIRVGlobalRegistry::getPointeeType(SPIRVType *PtrType) {1387  return PtrType && PtrType->getOpcode() == SPIRV::OpTypePointer1388             ? getSPIRVTypeForVReg(PtrType->getOperand(2).getReg())1389             : nullptr;1390}1391 1392unsigned SPIRVGlobalRegistry::getPointeeTypeOp(Register PtrReg) {1393  SPIRVType *ElemType = getPointeeType(getSPIRVTypeForVReg(PtrReg));1394  return ElemType ? ElemType->getOpcode() : 0;1395}1396 1397bool SPIRVGlobalRegistry::isBitcastCompatible(const SPIRVType *Type1,1398                                              const SPIRVType *Type2) const {1399  if (!Type1 || !Type2)1400    return false;1401  auto Op1 = Type1->getOpcode(), Op2 = Type2->getOpcode();1402  // Ignore difference between <1.5 and >=1.5 protocol versions:1403  // it's valid if either Result Type or Operand is a pointer, and the other1404  // is a pointer, an integer scalar, or an integer vector.1405  if (Op1 == SPIRV::OpTypePointer &&1406      (Op2 == SPIRV::OpTypePointer || retrieveScalarOrVectorIntType(Type2)))1407    return true;1408  if (Op2 == SPIRV::OpTypePointer &&1409      (Op1 == SPIRV::OpTypePointer || retrieveScalarOrVectorIntType(Type1)))1410    return true;1411  unsigned Bits1 = getNumScalarOrVectorTotalBitWidth(Type1),1412           Bits2 = getNumScalarOrVectorTotalBitWidth(Type2);1413  return Bits1 > 0 && Bits1 == Bits2;1414}1415 1416SPIRV::StorageClass::StorageClass1417SPIRVGlobalRegistry::getPointerStorageClass(Register VReg) const {1418  SPIRVType *Type = getSPIRVTypeForVReg(VReg);1419  assert(Type && Type->getOpcode() == SPIRV::OpTypePointer &&1420         Type->getOperand(1).isImm() && "Pointer type is expected");1421  return getPointerStorageClass(Type);1422}1423 1424SPIRV::StorageClass::StorageClass1425SPIRVGlobalRegistry::getPointerStorageClass(const SPIRVType *Type) const {1426  return static_cast<SPIRV::StorageClass::StorageClass>(1427      Type->getOperand(1).getImm());1428}1429 1430SPIRVType *SPIRVGlobalRegistry::getOrCreateVulkanBufferType(1431    MachineIRBuilder &MIRBuilder, Type *ElemType,1432    SPIRV::StorageClass::StorageClass SC, bool IsWritable, bool EmitIr) {1433  auto Key = SPIRV::irhandle_vkbuffer(ElemType, SC, IsWritable);1434  if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))1435    return MI;1436 1437  bool ExplicitLayoutRequired = storageClassRequiresExplictLayout(SC);1438  // We need to get the SPIR-V type for the element here, so we can add the1439  // decoration to it.1440  auto *T = StructType::create(ElemType);1441  auto *BlockType =1442      getOrCreateSPIRVType(T, MIRBuilder, SPIRV::AccessQualifier::None,1443                           ExplicitLayoutRequired, EmitIr);1444 1445  buildOpDecorate(BlockType->defs().begin()->getReg(), MIRBuilder,1446                  SPIRV::Decoration::Block, {});1447 1448  if (!IsWritable) {1449    buildOpMemberDecorate(BlockType->defs().begin()->getReg(), MIRBuilder,1450                          SPIRV::Decoration::NonWritable, 0, {});1451  }1452 1453  SPIRVType *R = getOrCreateSPIRVPointerTypeInternal(BlockType, MIRBuilder, SC);1454  add(Key, R);1455  return R;1456}1457 1458SPIRVType *1459SPIRVGlobalRegistry::getOrCreatePaddingType(MachineIRBuilder &MIRBuilder) {1460  auto Key = SPIRV::irhandle_padding();1461  if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))1462    return MI;1463  auto *T = Type::getInt8Ty(MIRBuilder.getContext());1464  SPIRVType *R = getOrCreateSPIRVIntegerType(8, MIRBuilder);1465  finishCreatingSPIRVType(T, R);1466  add(Key, R);1467  return R;1468}1469 1470SPIRVType *SPIRVGlobalRegistry::getOrCreateLayoutType(1471    MachineIRBuilder &MIRBuilder, const TargetExtType *T, bool EmitIr) {1472  auto Key = SPIRV::handle(T);1473  if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))1474    return MI;1475 1476  StructType *ST = cast<StructType>(T->getTypeParameter(0));1477  ArrayRef<uint32_t> Offsets = T->int_params().slice(1);1478  assert(ST->getNumElements() == Offsets.size());1479 1480  StructOffsetDecorator Decorator = [&MIRBuilder, &Offsets](Register Reg) {1481    for (uint32_t I = 0; I < Offsets.size(); ++I) {1482      buildOpMemberDecorate(Reg, MIRBuilder, SPIRV::Decoration::Offset, I,1483                            {Offsets[I]});1484    }1485  };1486 1487  // We need a new OpTypeStruct instruction because decorations will be1488  // different from a struct with an explicit layout created from a different1489  // entry point.1490  SPIRVType *SPIRVStructType =1491      getOpTypeStruct(ST, MIRBuilder, SPIRV::AccessQualifier::None,1492                      std::move(Decorator), EmitIr);1493  add(Key, SPIRVStructType);1494  return SPIRVStructType;1495}1496 1497SPIRVType *SPIRVGlobalRegistry::getImageType(1498    const TargetExtType *ExtensionType,1499    const SPIRV::AccessQualifier::AccessQualifier Qualifier,1500    MachineIRBuilder &MIRBuilder) {1501  assert(ExtensionType->getNumTypeParameters() == 1 &&1502         "SPIR-V image builtin type must have sampled type parameter!");1503  const SPIRVType *SampledType =1504      getOrCreateSPIRVType(ExtensionType->getTypeParameter(0), MIRBuilder,1505                           SPIRV::AccessQualifier::ReadWrite, true);1506  assert((ExtensionType->getNumIntParameters() == 7 ||1507          ExtensionType->getNumIntParameters() == 6) &&1508         "Invalid number of parameters for SPIR-V image builtin!");1509 1510  SPIRV::AccessQualifier::AccessQualifier accessQualifier =1511      SPIRV::AccessQualifier::None;1512  if (ExtensionType->getNumIntParameters() == 7) {1513    accessQualifier = Qualifier == SPIRV::AccessQualifier::WriteOnly1514                          ? SPIRV::AccessQualifier::WriteOnly1515                          : SPIRV::AccessQualifier::AccessQualifier(1516                                ExtensionType->getIntParameter(6));1517  }1518 1519  // Create or get an existing type from GlobalRegistry.1520  SPIRVType *R = getOrCreateOpTypeImage(1521      MIRBuilder, SampledType,1522      SPIRV::Dim::Dim(ExtensionType->getIntParameter(0)),1523      ExtensionType->getIntParameter(1), ExtensionType->getIntParameter(2),1524      ExtensionType->getIntParameter(3), ExtensionType->getIntParameter(4),1525      SPIRV::ImageFormat::ImageFormat(ExtensionType->getIntParameter(5)),1526      accessQualifier);1527  SPIRVToLLVMType[R] = ExtensionType;1528  return R;1529}1530 1531SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeImage(1532    MachineIRBuilder &MIRBuilder, SPIRVType *SampledType, SPIRV::Dim::Dim Dim,1533    uint32_t Depth, uint32_t Arrayed, uint32_t Multisampled, uint32_t Sampled,1534    SPIRV::ImageFormat::ImageFormat ImageFormat,1535    SPIRV::AccessQualifier::AccessQualifier AccessQual) {1536  auto Key = SPIRV::irhandle_image(SPIRVToLLVMType.lookup(SampledType), Dim,1537                                   Depth, Arrayed, Multisampled, Sampled,1538                                   ImageFormat, AccessQual);1539  if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))1540    return MI;1541  const MachineInstr *NewMI =1542      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1543        auto MIB =1544            MIRBuilder.buildInstr(SPIRV::OpTypeImage)1545                .addDef(createTypeVReg(MIRBuilder))1546                .addUse(getSPIRVTypeID(SampledType))1547                .addImm(Dim)1548                .addImm(Depth)   // Depth (whether or not it is a Depth image).1549                .addImm(Arrayed) // Arrayed.1550                .addImm(Multisampled) // Multisampled (0 = only single-sample).1551                .addImm(Sampled)      // Sampled (0 = usage known at runtime).1552                .addImm(ImageFormat);1553        if (AccessQual != SPIRV::AccessQualifier::None)1554          MIB.addImm(AccessQual);1555        return MIB;1556      });1557  add(Key, NewMI);1558  return NewMI;1559}1560 1561SPIRVType *1562SPIRVGlobalRegistry::getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder) {1563  auto Key = SPIRV::irhandle_sampler();1564  const MachineFunction *MF = &MIRBuilder.getMF();1565  if (const MachineInstr *MI = findMI(Key, MF))1566    return MI;1567  const MachineInstr *NewMI =1568      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1569        return MIRBuilder.buildInstr(SPIRV::OpTypeSampler)1570            .addDef(createTypeVReg(MIRBuilder));1571      });1572  add(Key, NewMI);1573  return NewMI;1574}1575 1576SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypePipe(1577    MachineIRBuilder &MIRBuilder,1578    SPIRV::AccessQualifier::AccessQualifier AccessQual) {1579  auto Key = SPIRV::irhandle_pipe(AccessQual);1580  if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))1581    return MI;1582  const MachineInstr *NewMI =1583      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1584        return MIRBuilder.buildInstr(SPIRV::OpTypePipe)1585            .addDef(createTypeVReg(MIRBuilder))1586            .addImm(AccessQual);1587      });1588  add(Key, NewMI);1589  return NewMI;1590}1591 1592SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeDeviceEvent(1593    MachineIRBuilder &MIRBuilder) {1594  auto Key = SPIRV::irhandle_event();1595  if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))1596    return MI;1597  const MachineInstr *NewMI =1598      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1599        return MIRBuilder.buildInstr(SPIRV::OpTypeDeviceEvent)1600            .addDef(createTypeVReg(MIRBuilder));1601      });1602  add(Key, NewMI);1603  return NewMI;1604}1605 1606SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeSampledImage(1607    SPIRVType *ImageType, MachineIRBuilder &MIRBuilder) {1608  auto Key = SPIRV::irhandle_sampled_image(1609      SPIRVToLLVMType.lookup(MIRBuilder.getMF().getRegInfo().getVRegDef(1610          ImageType->getOperand(1).getReg())),1611      ImageType);1612  if (const MachineInstr *MI = findMI(Key, &MIRBuilder.getMF()))1613    return MI;1614  const MachineInstr *NewMI =1615      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1616        return MIRBuilder.buildInstr(SPIRV::OpTypeSampledImage)1617            .addDef(createTypeVReg(MIRBuilder))1618            .addUse(getSPIRVTypeID(ImageType));1619      });1620  add(Key, NewMI);1621  return NewMI;1622}1623 1624SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeCoopMatr(1625    MachineIRBuilder &MIRBuilder, const TargetExtType *ExtensionType,1626    const SPIRVType *ElemType, uint32_t Scope, uint32_t Rows, uint32_t Columns,1627    uint32_t Use, bool EmitIR) {1628  if (const MachineInstr *MI =1629          findMI(ExtensionType, false, &MIRBuilder.getMF()))1630    return MI;1631  const MachineInstr *NewMI =1632      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1633        SPIRVType *SpvTypeInt32 = getOrCreateSPIRVIntegerType(32, MIRBuilder);1634        const Type *ET = getTypeForSPIRVType(ElemType);1635        if (ET->isIntegerTy() && ET->getIntegerBitWidth() == 4 &&1636            cast<SPIRVSubtarget>(MIRBuilder.getMF().getSubtarget())1637                .canUseExtension(SPIRV::Extension::SPV_INTEL_int4)) {1638          MIRBuilder.buildInstr(SPIRV::OpCapability)1639              .addImm(SPIRV::Capability::Int4CooperativeMatrixINTEL);1640        }1641        return MIRBuilder.buildInstr(SPIRV::OpTypeCooperativeMatrixKHR)1642            .addDef(createTypeVReg(MIRBuilder))1643            .addUse(getSPIRVTypeID(ElemType))1644            .addUse(buildConstantInt(Scope, MIRBuilder, SpvTypeInt32, EmitIR))1645            .addUse(buildConstantInt(Rows, MIRBuilder, SpvTypeInt32, EmitIR))1646            .addUse(buildConstantInt(Columns, MIRBuilder, SpvTypeInt32, EmitIR))1647            .addUse(buildConstantInt(Use, MIRBuilder, SpvTypeInt32, EmitIR));1648      });1649  add(ExtensionType, false, NewMI);1650  return NewMI;1651}1652 1653SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeByOpcode(1654    const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode) {1655  if (const MachineInstr *MI = findMI(Ty, false, &MIRBuilder.getMF()))1656    return MI;1657  const MachineInstr *NewMI =1658      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1659        return MIRBuilder.buildInstr(Opcode).addDef(createTypeVReg(MIRBuilder));1660      });1661  add(Ty, false, NewMI);1662  return NewMI;1663}1664 1665SPIRVType *SPIRVGlobalRegistry::getOrCreateUnknownType(1666    const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode,1667    const ArrayRef<MCOperand> Operands) {1668  if (const MachineInstr *MI = findMI(Ty, false, &MIRBuilder.getMF()))1669    return MI;1670  Register ResVReg = createTypeVReg(MIRBuilder);1671  const MachineInstr *NewMI =1672      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1673        MachineInstrBuilder MIB = MIRBuilder.buildInstr(SPIRV::UNKNOWN_type)1674                                      .addDef(ResVReg)1675                                      .addImm(Opcode);1676        for (MCOperand Operand : Operands) {1677          if (Operand.isReg()) {1678            MIB.addUse(Operand.getReg());1679          } else if (Operand.isImm()) {1680            MIB.addImm(Operand.getImm());1681          }1682        }1683        return MIB;1684      });1685  add(Ty, false, NewMI);1686  return NewMI;1687}1688 1689// Returns nullptr if unable to recognize SPIRV type name1690SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVTypeByName(1691    StringRef TypeStr, MachineIRBuilder &MIRBuilder, bool EmitIR,1692    SPIRV::StorageClass::StorageClass SC,1693    SPIRV::AccessQualifier::AccessQualifier AQ) {1694  unsigned VecElts = 0;1695  auto &Ctx = MIRBuilder.getMF().getFunction().getContext();1696 1697  // Parse strings representing either a SPIR-V or OpenCL builtin type.1698  if (hasBuiltinTypePrefix(TypeStr))1699    return getOrCreateSPIRVType(SPIRV::parseBuiltinTypeNameToTargetExtType(1700                                    TypeStr.str(), MIRBuilder.getContext()),1701                                MIRBuilder, AQ, false, true);1702 1703  // Parse type name in either "typeN" or "type vector[N]" format, where1704  // N is the number of elements of the vector.1705  Type *Ty;1706 1707  Ty = parseBasicTypeName(TypeStr, Ctx);1708  if (!Ty)1709    // Unable to recognize SPIRV type name1710    return nullptr;1711 1712  const SPIRVType *SpirvTy =1713      getOrCreateSPIRVType(Ty, MIRBuilder, AQ, false, true);1714 1715  // Handle "type*" or  "type* vector[N]".1716  if (TypeStr.consume_front("*"))1717    SpirvTy = getOrCreateSPIRVPointerType(Ty, MIRBuilder, SC);1718 1719  // Handle "typeN*" or  "type vector[N]*".1720  bool IsPtrToVec = TypeStr.consume_back("*");1721 1722  if (TypeStr.consume_front(" vector[")) {1723    TypeStr = TypeStr.substr(0, TypeStr.find(']'));1724  }1725  TypeStr.getAsInteger(10, VecElts);1726  if (VecElts > 0)1727    SpirvTy = getOrCreateSPIRVVectorType(SpirvTy, VecElts, MIRBuilder, EmitIR);1728 1729  if (IsPtrToVec)1730    SpirvTy = getOrCreateSPIRVPointerType(SpirvTy, MIRBuilder, SC);1731 1732  return SpirvTy;1733}1734 1735SPIRVType *1736SPIRVGlobalRegistry::getOrCreateSPIRVIntegerType(unsigned BitWidth,1737                                                 MachineIRBuilder &MIRBuilder) {1738  return getOrCreateSPIRVType(1739      IntegerType::get(MIRBuilder.getMF().getFunction().getContext(), BitWidth),1740      MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false, true);1741}1742 1743SPIRVType *SPIRVGlobalRegistry::finishCreatingSPIRVType(const Type *LLVMTy,1744                                                        SPIRVType *SpirvType) {1745  assert(CurMF == SpirvType->getMF());1746  VRegToTypeMap[CurMF][getSPIRVTypeID(SpirvType)] = SpirvType;1747  SPIRVToLLVMType[SpirvType] = unifyPtrType(LLVMTy);1748  return SpirvType;1749}1750 1751SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVType(unsigned BitWidth,1752                                                     MachineInstr &I,1753                                                     const SPIRVInstrInfo &TII,1754                                                     unsigned SPIRVOPcode,1755                                                     Type *Ty) {1756  if (const MachineInstr *MI = findMI(Ty, false, CurMF))1757    return MI;1758  MachineBasicBlock &DepMBB = I.getMF()->front();1759  MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());1760  const MachineInstr *NewMI =1761      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1762        auto NewTypeMI = BuildMI(MIRBuilder.getMBB(), *MIRBuilder.getInsertPt(),1763                                 MIRBuilder.getDL(), TII.get(SPIRVOPcode))1764                             .addDef(createTypeVReg(CurMF->getRegInfo()))1765                             .addImm(BitWidth);1766        // Don't add Encoding to FP type1767        if (!Ty->isFloatTy()) {1768          return NewTypeMI.addImm(0);1769        } else {1770          return NewTypeMI;1771        }1772      });1773  add(Ty, false, NewMI);1774  return finishCreatingSPIRVType(Ty, NewMI);1775}1776 1777SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVIntegerType(1778    unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII) {1779  // Maybe adjust bit width to keep DuplicateTracker consistent. Without1780  // such an adjustment SPIRVGlobalRegistry::getOpTypeInt() could create, for1781  // example, the same "OpTypeInt 8" type for a series of LLVM integer types1782  // with number of bits less than 8, causing duplicate type definitions.1783  if (BitWidth > 1)1784    BitWidth = adjustOpTypeIntWidth(BitWidth);1785  Type *LLVMTy = IntegerType::get(CurMF->getFunction().getContext(), BitWidth);1786  return getOrCreateSPIRVType(BitWidth, I, TII, SPIRV::OpTypeInt, LLVMTy);1787}1788 1789SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVFloatType(1790    unsigned BitWidth, MachineInstr &I, const SPIRVInstrInfo &TII) {1791  LLVMContext &Ctx = CurMF->getFunction().getContext();1792  Type *LLVMTy;1793  switch (BitWidth) {1794  case 16:1795    LLVMTy = Type::getHalfTy(Ctx);1796    break;1797  case 32:1798    LLVMTy = Type::getFloatTy(Ctx);1799    break;1800  case 64:1801    LLVMTy = Type::getDoubleTy(Ctx);1802    break;1803  default:1804    llvm_unreachable("Bit width is of unexpected size.");1805  }1806  return getOrCreateSPIRVType(BitWidth, I, TII, SPIRV::OpTypeFloat, LLVMTy);1807}1808 1809SPIRVType *1810SPIRVGlobalRegistry::getOrCreateSPIRVBoolType(MachineIRBuilder &MIRBuilder,1811                                              bool EmitIR) {1812  return getOrCreateSPIRVType(1813      IntegerType::get(MIRBuilder.getMF().getFunction().getContext(), 1),1814      MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false, EmitIR);1815}1816 1817SPIRVType *1818SPIRVGlobalRegistry::getOrCreateSPIRVBoolType(MachineInstr &I,1819                                              const SPIRVInstrInfo &TII) {1820  Type *Ty = IntegerType::get(CurMF->getFunction().getContext(), 1);1821  if (const MachineInstr *MI = findMI(Ty, false, CurMF))1822    return MI;1823  MachineBasicBlock &DepMBB = I.getMF()->front();1824  MachineIRBuilder MIRBuilder(DepMBB, DepMBB.getFirstNonPHI());1825  const MachineInstr *NewMI =1826      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1827        return BuildMI(MIRBuilder.getMBB(), *MIRBuilder.getInsertPt(),1828                       MIRBuilder.getDL(), TII.get(SPIRV::OpTypeBool))1829            .addDef(createTypeVReg(CurMF->getRegInfo()));1830      });1831  add(Ty, false, NewMI);1832  return finishCreatingSPIRVType(Ty, NewMI);1833}1834 1835SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVVectorType(1836    SPIRVType *BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder,1837    bool EmitIR) {1838  return getOrCreateSPIRVType(1839      FixedVectorType::get(const_cast<Type *>(getTypeForSPIRVType(BaseType)),1840                           NumElements),1841      MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false, EmitIR);1842}1843 1844SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVVectorType(1845    SPIRVType *BaseType, unsigned NumElements, MachineInstr &I,1846    const SPIRVInstrInfo &TII) {1847  Type *Ty = FixedVectorType::get(1848      const_cast<Type *>(getTypeForSPIRVType(BaseType)), NumElements);1849  if (const MachineInstr *MI = findMI(Ty, false, CurMF))1850    return MI;1851  MachineInstr *DepMI = const_cast<MachineInstr *>(BaseType);1852  MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());1853  const MachineInstr *NewMI =1854      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1855        return BuildMI(MIRBuilder.getMBB(), *MIRBuilder.getInsertPt(),1856                       MIRBuilder.getDL(), TII.get(SPIRV::OpTypeVector))1857            .addDef(createTypeVReg(CurMF->getRegInfo()))1858            .addUse(getSPIRVTypeID(BaseType))1859            .addImm(NumElements);1860      });1861  add(Ty, false, NewMI);1862  return finishCreatingSPIRVType(Ty, NewMI);1863}1864 1865SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerType(1866    const Type *BaseType, MachineInstr &I,1867    SPIRV::StorageClass::StorageClass SC) {1868  MachineIRBuilder MIRBuilder(I);1869  return getOrCreateSPIRVPointerType(BaseType, MIRBuilder, SC);1870}1871 1872SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerType(1873    const Type *BaseType, MachineIRBuilder &MIRBuilder,1874    SPIRV::StorageClass::StorageClass SC) {1875  // TODO: Need to check if EmitIr should always be true.1876  SPIRVType *SpirvBaseType = getOrCreateSPIRVType(1877      BaseType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite,1878      storageClassRequiresExplictLayout(SC), true);1879  assert(SpirvBaseType);1880  return getOrCreateSPIRVPointerTypeInternal(SpirvBaseType, MIRBuilder, SC);1881}1882 1883SPIRVType *SPIRVGlobalRegistry::changePointerStorageClass(1884    SPIRVType *PtrType, SPIRV::StorageClass::StorageClass SC, MachineInstr &I) {1885  [[maybe_unused]] SPIRV::StorageClass::StorageClass OldSC =1886      getPointerStorageClass(PtrType);1887  assert(storageClassRequiresExplictLayout(OldSC) ==1888         storageClassRequiresExplictLayout(SC));1889 1890  SPIRVType *PointeeType = getPointeeType(PtrType);1891  MachineIRBuilder MIRBuilder(I);1892  return getOrCreateSPIRVPointerTypeInternal(PointeeType, MIRBuilder, SC);1893}1894 1895SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerType(1896    SPIRVType *BaseType, MachineIRBuilder &MIRBuilder,1897    SPIRV::StorageClass::StorageClass SC) {1898  const Type *LLVMType = getTypeForSPIRVType(BaseType);1899  assert(!storageClassRequiresExplictLayout(SC));1900  SPIRVType *R = getOrCreateSPIRVPointerType(LLVMType, MIRBuilder, SC);1901  assert(1902      getPointeeType(R) == BaseType &&1903      "The base type was not correctly laid out for the given storage class.");1904  return R;1905}1906 1907SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerTypeInternal(1908    SPIRVType *BaseType, MachineIRBuilder &MIRBuilder,1909    SPIRV::StorageClass::StorageClass SC) {1910  const Type *PointerElementType = getTypeForSPIRVType(BaseType);1911  unsigned AddressSpace = storageClassToAddressSpace(SC);1912  if (const MachineInstr *MI = findMI(PointerElementType, AddressSpace, CurMF))1913    return MI;1914  Type *Ty = TypedPointerType::get(const_cast<Type *>(PointerElementType),1915                                   AddressSpace);1916  const MachineInstr *NewMI =1917      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1918        return BuildMI(MIRBuilder.getMBB(), MIRBuilder.getInsertPt(),1919                       MIRBuilder.getDebugLoc(),1920                       MIRBuilder.getTII().get(SPIRV::OpTypePointer))1921            .addDef(createTypeVReg(CurMF->getRegInfo()))1922            .addImm(static_cast<uint32_t>(SC))1923            .addUse(getSPIRVTypeID(BaseType));1924      });1925  add(PointerElementType, AddressSpace, NewMI);1926  return finishCreatingSPIRVType(Ty, NewMI);1927}1928 1929Register SPIRVGlobalRegistry::getOrCreateUndef(MachineInstr &I,1930                                               SPIRVType *SpvType,1931                                               const SPIRVInstrInfo &TII) {1932  UndefValue *UV =1933      UndefValue::get(const_cast<Type *>(getTypeForSPIRVType(SpvType)));1934  Register Res = find(UV, CurMF);1935  if (Res.isValid())1936    return Res;1937 1938  LLT LLTy = LLT::scalar(64);1939  Res = CurMF->getRegInfo().createGenericVirtualRegister(LLTy);1940  CurMF->getRegInfo().setRegClass(Res, &SPIRV::iIDRegClass);1941  assignSPIRVTypeToVReg(SpvType, Res, *CurMF);1942 1943  MachineInstr *DepMI = const_cast<MachineInstr *>(SpvType);1944  MachineIRBuilder MIRBuilder(*DepMI->getParent(), DepMI->getIterator());1945  const MachineInstr *NewMI =1946      createOpType(MIRBuilder, [&](MachineIRBuilder &MIRBuilder) {1947        auto MIB = BuildMI(MIRBuilder.getMBB(), *MIRBuilder.getInsertPt(),1948                           MIRBuilder.getDL(), TII.get(SPIRV::OpUndef))1949                       .addDef(Res)1950                       .addUse(getSPIRVTypeID(SpvType));1951        const auto &ST = CurMF->getSubtarget();1952        constrainSelectedInstRegOperands(*MIB, *ST.getInstrInfo(),1953                                         *ST.getRegisterInfo(),1954                                         *ST.getRegBankInfo());1955        return MIB;1956      });1957  add(UV, NewMI);1958  return Res;1959}1960 1961const TargetRegisterClass *1962SPIRVGlobalRegistry::getRegClass(SPIRVType *SpvType) const {1963  unsigned Opcode = SpvType->getOpcode();1964  switch (Opcode) {1965  case SPIRV::OpTypeFloat:1966    return &SPIRV::fIDRegClass;1967  case SPIRV::OpTypePointer:1968    return &SPIRV::pIDRegClass;1969  case SPIRV::OpTypeVector: {1970    SPIRVType *ElemType = getSPIRVTypeForVReg(SpvType->getOperand(1).getReg());1971    unsigned ElemOpcode = ElemType ? ElemType->getOpcode() : 0;1972    if (ElemOpcode == SPIRV::OpTypeFloat)1973      return &SPIRV::vfIDRegClass;1974    if (ElemOpcode == SPIRV::OpTypePointer)1975      return &SPIRV::vpIDRegClass;1976    return &SPIRV::vIDRegClass;1977  }1978  }1979  return &SPIRV::iIDRegClass;1980}1981 1982inline unsigned getAS(SPIRVType *SpvType) {1983  return storageClassToAddressSpace(1984      static_cast<SPIRV::StorageClass::StorageClass>(1985          SpvType->getOperand(1).getImm()));1986}1987 1988LLT SPIRVGlobalRegistry::getRegType(SPIRVType *SpvType) const {1989  unsigned Opcode = SpvType ? SpvType->getOpcode() : 0;1990  switch (Opcode) {1991  case SPIRV::OpTypeInt:1992  case SPIRV::OpTypeFloat:1993  case SPIRV::OpTypeBool:1994    return LLT::scalar(getScalarOrVectorBitWidth(SpvType));1995  case SPIRV::OpTypePointer:1996    return LLT::pointer(getAS(SpvType), getPointerSize());1997  case SPIRV::OpTypeVector: {1998    SPIRVType *ElemType = getSPIRVTypeForVReg(SpvType->getOperand(1).getReg());1999    LLT ET;2000    switch (ElemType ? ElemType->getOpcode() : 0) {2001    case SPIRV::OpTypePointer:2002      ET = LLT::pointer(getAS(ElemType), getPointerSize());2003      break;2004    case SPIRV::OpTypeInt:2005    case SPIRV::OpTypeFloat:2006    case SPIRV::OpTypeBool:2007      ET = LLT::scalar(getScalarOrVectorBitWidth(ElemType));2008      break;2009    default:2010      ET = LLT::scalar(64);2011    }2012    return LLT::fixed_vector(2013        static_cast<unsigned>(SpvType->getOperand(2).getImm()), ET);2014  }2015  }2016  return LLT::scalar(64);2017}2018 2019// Aliasing list MD contains several scope MD nodes whithin it. Each scope MD2020// has a selfreference and an extra MD node for aliasing domain and also it2021// can contain an optional string operand. Domain MD contains a self-reference2022// with an optional string operand. Here we unfold the list, creating SPIR-V2023// aliasing instructions.2024// TODO: add support for an optional string operand.2025MachineInstr *SPIRVGlobalRegistry::getOrAddMemAliasingINTELInst(2026    MachineIRBuilder &MIRBuilder, const MDNode *AliasingListMD) {2027  if (AliasingListMD->getNumOperands() == 0)2028    return nullptr;2029  if (auto L = AliasInstMDMap.find(AliasingListMD); L != AliasInstMDMap.end())2030    return L->second;2031 2032  SmallVector<MachineInstr *> ScopeList;2033  MachineRegisterInfo *MRI = MIRBuilder.getMRI();2034  for (const MDOperand &MDListOp : AliasingListMD->operands()) {2035    if (MDNode *ScopeMD = dyn_cast<MDNode>(MDListOp)) {2036      if (ScopeMD->getNumOperands() < 2)2037        return nullptr;2038      MDNode *DomainMD = dyn_cast<MDNode>(ScopeMD->getOperand(1));2039      if (!DomainMD)2040        return nullptr;2041      auto *Domain = [&] {2042        auto D = AliasInstMDMap.find(DomainMD);2043        if (D != AliasInstMDMap.end())2044          return D->second;2045        const Register Ret = MRI->createVirtualRegister(&SPIRV::IDRegClass);2046        auto MIB =2047            MIRBuilder.buildInstr(SPIRV::OpAliasDomainDeclINTEL).addDef(Ret);2048        return MIB.getInstr();2049      }();2050      AliasInstMDMap.insert(std::make_pair(DomainMD, Domain));2051      auto *Scope = [&] {2052        auto S = AliasInstMDMap.find(ScopeMD);2053        if (S != AliasInstMDMap.end())2054          return S->second;2055        const Register Ret = MRI->createVirtualRegister(&SPIRV::IDRegClass);2056        auto MIB = MIRBuilder.buildInstr(SPIRV::OpAliasScopeDeclINTEL)2057                       .addDef(Ret)2058                       .addUse(Domain->getOperand(0).getReg());2059        return MIB.getInstr();2060      }();2061      AliasInstMDMap.insert(std::make_pair(ScopeMD, Scope));2062      ScopeList.push_back(Scope);2063    }2064  }2065 2066  const Register Ret = MRI->createVirtualRegister(&SPIRV::IDRegClass);2067  auto MIB =2068      MIRBuilder.buildInstr(SPIRV::OpAliasScopeListDeclINTEL).addDef(Ret);2069  for (auto *Scope : ScopeList)2070    MIB.addUse(Scope->getOperand(0).getReg());2071  auto List = MIB.getInstr();2072  AliasInstMDMap.insert(std::make_pair(AliasingListMD, List));2073  return List;2074}2075 2076void SPIRVGlobalRegistry::buildMemAliasingOpDecorate(2077    Register Reg, MachineIRBuilder &MIRBuilder, uint32_t Dec,2078    const MDNode *AliasingListMD) {2079  MachineInstr *AliasList =2080      getOrAddMemAliasingINTELInst(MIRBuilder, AliasingListMD);2081  if (!AliasList)2082    return;2083  MIRBuilder.buildInstr(SPIRV::OpDecorate)2084      .addUse(Reg)2085      .addImm(Dec)2086      .addUse(AliasList->getOperand(0).getReg());2087}2088void SPIRVGlobalRegistry::replaceAllUsesWith(Value *Old, Value *New,2089                                             bool DeleteOld) {2090  Old->replaceAllUsesWith(New);2091  updateIfExistDeducedElementType(Old, New, DeleteOld);2092  updateIfExistAssignPtrTypeInstr(Old, New, DeleteOld);2093}2094 2095void SPIRVGlobalRegistry::buildAssignType(IRBuilder<> &B, Type *Ty,2096                                          Value *Arg) {2097  Value *OfType = getNormalizedPoisonValue(Ty);2098  CallInst *AssignCI = nullptr;2099  if (Arg->getType()->isAggregateType() && Ty->isAggregateType() &&2100      allowEmitFakeUse(Arg)) {2101    LLVMContext &Ctx = Arg->getContext();2102    SmallVector<Metadata *, 2> ArgMDs{2103        MDNode::get(Ctx, ValueAsMetadata::getConstant(OfType)),2104        MDString::get(Ctx, Arg->getName())};2105    B.CreateIntrinsic(Intrinsic::spv_value_md,2106                      {MetadataAsValue::get(Ctx, MDTuple::get(Ctx, ArgMDs))});2107    AssignCI = B.CreateIntrinsic(Intrinsic::fake_use, {Arg});2108  } else {2109    AssignCI = buildIntrWithMD(Intrinsic::spv_assign_type, {Arg->getType()},2110                               OfType, Arg, {}, B);2111  }2112  addAssignPtrTypeInstr(Arg, AssignCI);2113}2114 2115void SPIRVGlobalRegistry::buildAssignPtr(IRBuilder<> &B, Type *ElemTy,2116                                         Value *Arg) {2117  Value *OfType = PoisonValue::get(ElemTy);2118  CallInst *AssignPtrTyCI = findAssignPtrTypeInstr(Arg);2119  Function *CurrF =2120      B.GetInsertBlock() ? B.GetInsertBlock()->getParent() : nullptr;2121  if (AssignPtrTyCI == nullptr ||2122      AssignPtrTyCI->getParent()->getParent() != CurrF) {2123    AssignPtrTyCI = buildIntrWithMD(2124        Intrinsic::spv_assign_ptr_type, {Arg->getType()}, OfType, Arg,2125        {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B);2126    addDeducedElementType(AssignPtrTyCI, ElemTy);2127    addDeducedElementType(Arg, ElemTy);2128    addAssignPtrTypeInstr(Arg, AssignPtrTyCI);2129  } else {2130    updateAssignType(AssignPtrTyCI, Arg, OfType);2131  }2132}2133 2134void SPIRVGlobalRegistry::updateAssignType(CallInst *AssignCI, Value *Arg,2135                                           Value *OfType) {2136  AssignCI->setArgOperand(1, buildMD(OfType));2137  if (cast<IntrinsicInst>(AssignCI)->getIntrinsicID() !=2138      Intrinsic::spv_assign_ptr_type)2139    return;2140 2141  // update association with the pointee type2142  Type *ElemTy = OfType->getType();2143  addDeducedElementType(AssignCI, ElemTy);2144  addDeducedElementType(Arg, ElemTy);2145}2146 2147void SPIRVGlobalRegistry::addStructOffsetDecorations(2148    Register Reg, StructType *Ty, MachineIRBuilder &MIRBuilder) {2149  DataLayout DL;2150  ArrayRef<TypeSize> Offsets = DL.getStructLayout(Ty)->getMemberOffsets();2151  for (uint32_t I = 0; I < Ty->getNumElements(); ++I) {2152    buildOpMemberDecorate(Reg, MIRBuilder, SPIRV::Decoration::Offset, I,2153                          {static_cast<uint32_t>(Offsets[I])});2154  }2155}2156 2157void SPIRVGlobalRegistry::addArrayStrideDecorations(2158    Register Reg, Type *ElementType, MachineIRBuilder &MIRBuilder) {2159  uint32_t SizeInBytes = DataLayout().getTypeSizeInBits(ElementType) / 8;2160  buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::ArrayStride,2161                  {SizeInBytes});2162}2163 2164bool SPIRVGlobalRegistry::hasBlockDecoration(SPIRVType *Type) const {2165  Register Def = getSPIRVTypeID(Type);2166  for (const MachineInstr &Use :2167       Type->getMF()->getRegInfo().use_instructions(Def)) {2168    if (Use.getOpcode() != SPIRV::OpDecorate)2169      continue;2170 2171    if (Use.getOperand(1).getImm() == SPIRV::Decoration::Block)2172      return true;2173  }2174  return false;2175}2176