2659 lines · cpp
1//===- SPIRVModuleAnalysis.cpp - analysis of global instrs & regs - C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// The analysis collects instructions that should be output at the module level10// and performs the global register numbering.11//12// The results of this analysis are used in AsmPrinter to rename registers13// globally and to output required instructions at the module level.14//15//===----------------------------------------------------------------------===//16 17#include "SPIRVModuleAnalysis.h"18#include "MCTargetDesc/SPIRVBaseInfo.h"19#include "MCTargetDesc/SPIRVMCTargetDesc.h"20#include "SPIRV.h"21#include "SPIRVSubtarget.h"22#include "SPIRVTargetMachine.h"23#include "SPIRVUtils.h"24#include "llvm/ADT/STLExtras.h"25#include "llvm/CodeGen/MachineModuleInfo.h"26#include "llvm/CodeGen/TargetPassConfig.h"27 28using namespace llvm;29 30#define DEBUG_TYPE "spirv-module-analysis"31 32static cl::opt<bool>33 SPVDumpDeps("spv-dump-deps",34 cl::desc("Dump MIR with SPIR-V dependencies info"),35 cl::Optional, cl::init(false));36 37static cl::list<SPIRV::Capability::Capability>38 AvoidCapabilities("avoid-spirv-capabilities",39 cl::desc("SPIR-V capabilities to avoid if there are "40 "other options enabling a feature"),41 cl::ZeroOrMore, cl::Hidden,42 cl::values(clEnumValN(SPIRV::Capability::Shader, "Shader",43 "SPIR-V Shader capability")));44// Use sets instead of cl::list to check "if contains" condition45struct AvoidCapabilitiesSet {46 SmallSet<SPIRV::Capability::Capability, 4> S;47 AvoidCapabilitiesSet() { S.insert_range(AvoidCapabilities); }48};49 50char llvm::SPIRVModuleAnalysis::ID = 0;51 52INITIALIZE_PASS(SPIRVModuleAnalysis, DEBUG_TYPE, "SPIRV module analysis", true,53 true)54 55// Retrieve an unsigned from an MDNode with a list of them as operands.56static unsigned getMetadataUInt(MDNode *MdNode, unsigned OpIndex,57 unsigned DefaultVal = 0) {58 if (MdNode && OpIndex < MdNode->getNumOperands()) {59 const auto &Op = MdNode->getOperand(OpIndex);60 return mdconst::extract<ConstantInt>(Op)->getZExtValue();61 }62 return DefaultVal;63}64 65static SPIRV::Requirements66getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,67 unsigned i, const SPIRVSubtarget &ST,68 SPIRV::RequirementHandler &Reqs) {69 // A set of capabilities to avoid if there is another option.70 AvoidCapabilitiesSet AvoidCaps;71 if (!ST.isShader())72 AvoidCaps.S.insert(SPIRV::Capability::Shader);73 else74 AvoidCaps.S.insert(SPIRV::Capability::Kernel);75 76 VersionTuple ReqMinVer = getSymbolicOperandMinVersion(Category, i);77 VersionTuple ReqMaxVer = getSymbolicOperandMaxVersion(Category, i);78 VersionTuple SPIRVVersion = ST.getSPIRVVersion();79 bool MinVerOK = SPIRVVersion.empty() || SPIRVVersion >= ReqMinVer;80 bool MaxVerOK =81 ReqMaxVer.empty() || SPIRVVersion.empty() || SPIRVVersion <= ReqMaxVer;82 CapabilityList ReqCaps = getSymbolicOperandCapabilities(Category, i);83 ExtensionList ReqExts = getSymbolicOperandExtensions(Category, i);84 if (ReqCaps.empty()) {85 if (ReqExts.empty()) {86 if (MinVerOK && MaxVerOK)87 return {true, {}, {}, ReqMinVer, ReqMaxVer};88 return {false, {}, {}, VersionTuple(), VersionTuple()};89 }90 } else if (MinVerOK && MaxVerOK) {91 if (ReqCaps.size() == 1) {92 auto Cap = ReqCaps[0];93 if (Reqs.isCapabilityAvailable(Cap)) {94 ReqExts.append(getSymbolicOperandExtensions(95 SPIRV::OperandCategory::CapabilityOperand, Cap));96 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};97 }98 } else {99 // By SPIR-V specification: "If an instruction, enumerant, or other100 // feature specifies multiple enabling capabilities, only one such101 // capability needs to be declared to use the feature." However, one102 // capability may be preferred over another. We use command line103 // argument(s) and AvoidCapabilities to avoid selection of certain104 // capabilities if there are other options.105 CapabilityList UseCaps;106 for (auto Cap : ReqCaps)107 if (Reqs.isCapabilityAvailable(Cap))108 UseCaps.push_back(Cap);109 for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {110 auto Cap = UseCaps[i];111 if (i == Sz - 1 || !AvoidCaps.S.contains(Cap)) {112 ReqExts.append(getSymbolicOperandExtensions(113 SPIRV::OperandCategory::CapabilityOperand, Cap));114 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};115 }116 }117 }118 }119 // If there are no capabilities, or we can't satisfy the version or120 // capability requirements, use the list of extensions (if the subtarget121 // can handle them all).122 if (llvm::all_of(ReqExts, [&ST](const SPIRV::Extension::Extension &Ext) {123 return ST.canUseExtension(Ext);124 })) {125 return {true,126 {},127 std::move(ReqExts),128 VersionTuple(),129 VersionTuple()}; // TODO: add versions to extensions.130 }131 return {false, {}, {}, VersionTuple(), VersionTuple()};132}133 134void SPIRVModuleAnalysis::setBaseInfo(const Module &M) {135 MAI.MaxID = 0;136 for (int i = 0; i < SPIRV::NUM_MODULE_SECTIONS; i++)137 MAI.MS[i].clear();138 MAI.RegisterAliasTable.clear();139 MAI.InstrsToDelete.clear();140 MAI.FuncMap.clear();141 MAI.GlobalVarList.clear();142 MAI.ExtInstSetMap.clear();143 MAI.Reqs.clear();144 MAI.Reqs.initAvailableCapabilities(*ST);145 146 // TODO: determine memory model and source language from the configuratoin.147 if (auto MemModel = M.getNamedMetadata("spirv.MemoryModel")) {148 auto MemMD = MemModel->getOperand(0);149 MAI.Addr = static_cast<SPIRV::AddressingModel::AddressingModel>(150 getMetadataUInt(MemMD, 0));151 MAI.Mem =152 static_cast<SPIRV::MemoryModel::MemoryModel>(getMetadataUInt(MemMD, 1));153 } else {154 // TODO: Add support for VulkanMemoryModel.155 MAI.Mem = ST->isShader() ? SPIRV::MemoryModel::GLSL450156 : SPIRV::MemoryModel::OpenCL;157 if (MAI.Mem == SPIRV::MemoryModel::OpenCL) {158 unsigned PtrSize = ST->getPointerSize();159 MAI.Addr = PtrSize == 32 ? SPIRV::AddressingModel::Physical32160 : PtrSize == 64 ? SPIRV::AddressingModel::Physical64161 : SPIRV::AddressingModel::Logical;162 } else {163 // TODO: Add support for PhysicalStorageBufferAddress.164 MAI.Addr = SPIRV::AddressingModel::Logical;165 }166 }167 // Get the OpenCL version number from metadata.168 // TODO: support other source languages.169 if (auto VerNode = M.getNamedMetadata("opencl.ocl.version")) {170 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_C;171 // Construct version literal in accordance with SPIRV-LLVM-Translator.172 // TODO: support multiple OCL version metadata.173 assert(VerNode->getNumOperands() > 0 && "Invalid SPIR");174 auto VersionMD = VerNode->getOperand(0);175 unsigned MajorNum = getMetadataUInt(VersionMD, 0, 2);176 unsigned MinorNum = getMetadataUInt(VersionMD, 1);177 unsigned RevNum = getMetadataUInt(VersionMD, 2);178 // Prevent Major part of OpenCL version to be 0179 MAI.SrcLangVersion =180 (std::max(1U, MajorNum) * 100 + MinorNum) * 1000 + RevNum;181 } else {182 // If there is no information about OpenCL version we are forced to generate183 // OpenCL 1.0 by default for the OpenCL environment to avoid puzzling184 // run-times with Unknown/0.0 version output. For a reference, LLVM-SPIRV185 // Translator avoids potential issues with run-times in a similar manner.186 if (!ST->isShader()) {187 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_CPP;188 MAI.SrcLangVersion = 100000;189 } else {190 MAI.SrcLang = SPIRV::SourceLanguage::Unknown;191 MAI.SrcLangVersion = 0;192 }193 }194 195 if (auto ExtNode = M.getNamedMetadata("opencl.used.extensions")) {196 for (unsigned I = 0, E = ExtNode->getNumOperands(); I != E; ++I) {197 MDNode *MD = ExtNode->getOperand(I);198 if (!MD || MD->getNumOperands() == 0)199 continue;200 for (unsigned J = 0, N = MD->getNumOperands(); J != N; ++J)201 MAI.SrcExt.insert(cast<MDString>(MD->getOperand(J))->getString());202 }203 }204 205 // Update required capabilities for this memory model, addressing model and206 // source language.207 MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::MemoryModelOperand,208 MAI.Mem, *ST);209 MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::SourceLanguageOperand,210 MAI.SrcLang, *ST);211 MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::AddressingModelOperand,212 MAI.Addr, *ST);213 214 if (!ST->isShader()) {215 // TODO: check if it's required by default.216 MAI.ExtInstSetMap[static_cast<unsigned>(217 SPIRV::InstructionSet::OpenCL_std)] = MAI.getNextIDRegister();218 }219}220 221// Appends the signature of the decoration instructions that decorate R to222// Signature.223static void appendDecorationsForReg(const MachineRegisterInfo &MRI, Register R,224 InstrSignature &Signature) {225 for (MachineInstr &UseMI : MRI.use_instructions(R)) {226 // We don't handle OpDecorateId because getting the register alias for the227 // ID can cause problems, and we do not need it for now.228 if (UseMI.getOpcode() != SPIRV::OpDecorate &&229 UseMI.getOpcode() != SPIRV::OpMemberDecorate)230 continue;231 232 for (unsigned I = 0; I < UseMI.getNumOperands(); ++I) {233 const MachineOperand &MO = UseMI.getOperand(I);234 if (MO.isReg())235 continue;236 Signature.push_back(hash_value(MO));237 }238 }239}240 241// Returns a representation of an instruction as a vector of MachineOperand242// hash values, see llvm::hash_value(const MachineOperand &MO) for details.243// This creates a signature of the instruction with the same content244// that MachineOperand::isIdenticalTo uses for comparison.245static InstrSignature instrToSignature(const MachineInstr &MI,246 SPIRV::ModuleAnalysisInfo &MAI,247 bool UseDefReg) {248 Register DefReg;249 InstrSignature Signature{MI.getOpcode()};250 for (unsigned i = 0; i < MI.getNumOperands(); ++i) {251 // The only decorations that can be applied more than once to a given <id>252 // or structure member are FuncParamAttr (38), UserSemantic (5635),253 // CacheControlLoadINTEL (6442), and CacheControlStoreINTEL (6443). For all254 // the rest of decorations, we will only add to the signature the Opcode,255 // the id to which it applies, and the decoration id, disregarding any256 // decoration flags. This will ensure that any subsequent decoration with257 // the same id will be deemed as a duplicate. Then, at the call site, we258 // will be able to handle duplicates in the best way.259 unsigned Opcode = MI.getOpcode();260 if ((Opcode == SPIRV::OpDecorate) && i >= 2) {261 unsigned DecorationID = MI.getOperand(1).getImm();262 if (DecorationID != SPIRV::Decoration::FuncParamAttr &&263 DecorationID != SPIRV::Decoration::UserSemantic &&264 DecorationID != SPIRV::Decoration::CacheControlLoadINTEL &&265 DecorationID != SPIRV::Decoration::CacheControlStoreINTEL)266 continue;267 }268 const MachineOperand &MO = MI.getOperand(i);269 size_t h;270 if (MO.isReg()) {271 if (!UseDefReg && MO.isDef()) {272 assert(!DefReg.isValid() && "Multiple def registers.");273 DefReg = MO.getReg();274 continue;275 }276 Register RegAlias = MAI.getRegisterAlias(MI.getMF(), MO.getReg());277 if (!RegAlias.isValid()) {278 LLVM_DEBUG({279 dbgs() << "Unexpectedly, no global id found for the operand ";280 MO.print(dbgs());281 dbgs() << "\nInstruction: ";282 MI.print(dbgs());283 dbgs() << "\n";284 });285 report_fatal_error("All v-regs must have been mapped to global id's");286 }287 // mimic llvm::hash_value(const MachineOperand &MO)288 h = hash_combine(MO.getType(), (unsigned)RegAlias, MO.getSubReg(),289 MO.isDef());290 } else {291 h = hash_value(MO);292 }293 Signature.push_back(h);294 }295 296 if (DefReg.isValid()) {297 // Decorations change the semantics of the current instruction. So two298 // identical instruction with different decorations cannot be merged. That299 // is why we add the decorations to the signature.300 appendDecorationsForReg(MI.getMF()->getRegInfo(), DefReg, Signature);301 }302 return Signature;303}304 305bool SPIRVModuleAnalysis::isDeclSection(const MachineRegisterInfo &MRI,306 const MachineInstr &MI) {307 unsigned Opcode = MI.getOpcode();308 switch (Opcode) {309 case SPIRV::OpTypeForwardPointer:310 // omit now, collect later311 return false;312 case SPIRV::OpVariable:313 return static_cast<SPIRV::StorageClass::StorageClass>(314 MI.getOperand(2).getImm()) != SPIRV::StorageClass::Function;315 case SPIRV::OpFunction:316 case SPIRV::OpFunctionParameter:317 return true;318 }319 if (GR->hasConstFunPtr() && Opcode == SPIRV::OpUndef) {320 Register DefReg = MI.getOperand(0).getReg();321 for (MachineInstr &UseMI : MRI.use_instructions(DefReg)) {322 if (UseMI.getOpcode() != SPIRV::OpConstantFunctionPointerINTEL)323 continue;324 // it's a dummy definition, FP constant refers to a function,325 // and this is resolved in another way; let's skip this definition326 assert(UseMI.getOperand(2).isReg() &&327 UseMI.getOperand(2).getReg() == DefReg);328 MAI.setSkipEmission(&MI);329 return false;330 }331 }332 return TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||333 TII->isInlineAsmDefInstr(MI);334}335 336// This is a special case of a function pointer refering to a possibly337// forward function declaration. The operand is a dummy OpUndef that338// requires a special treatment.339void SPIRVModuleAnalysis::visitFunPtrUse(340 Register OpReg, InstrGRegsMap &SignatureToGReg,341 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,342 const MachineInstr &MI) {343 const MachineOperand *OpFunDef =344 GR->getFunctionDefinitionByUse(&MI.getOperand(2));345 assert(OpFunDef && OpFunDef->isReg());346 // find the actual function definition and number it globally in advance347 const MachineInstr *OpDefMI = OpFunDef->getParent();348 assert(OpDefMI && OpDefMI->getOpcode() == SPIRV::OpFunction);349 const MachineFunction *FunDefMF = OpDefMI->getParent()->getParent();350 const MachineRegisterInfo &FunDefMRI = FunDefMF->getRegInfo();351 do {352 visitDecl(FunDefMRI, SignatureToGReg, GlobalToGReg, FunDefMF, *OpDefMI);353 OpDefMI = OpDefMI->getNextNode();354 } while (OpDefMI && (OpDefMI->getOpcode() == SPIRV::OpFunction ||355 OpDefMI->getOpcode() == SPIRV::OpFunctionParameter));356 // associate the function pointer with the newly assigned global number357 MCRegister GlobalFunDefReg =358 MAI.getRegisterAlias(FunDefMF, OpFunDef->getReg());359 assert(GlobalFunDefReg.isValid() &&360 "Function definition must refer to a global register");361 MAI.setRegisterAlias(MF, OpReg, GlobalFunDefReg);362}363 364// Depth first recursive traversal of dependencies. Repeated visits are guarded365// by MAI.hasRegisterAlias().366void SPIRVModuleAnalysis::visitDecl(367 const MachineRegisterInfo &MRI, InstrGRegsMap &SignatureToGReg,368 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,369 const MachineInstr &MI) {370 unsigned Opcode = MI.getOpcode();371 372 // Process each operand of the instruction to resolve dependencies373 for (const MachineOperand &MO : MI.operands()) {374 if (!MO.isReg() || MO.isDef())375 continue;376 Register OpReg = MO.getReg();377 // Handle function pointers special case378 if (Opcode == SPIRV::OpConstantFunctionPointerINTEL &&379 MRI.getRegClass(OpReg) == &SPIRV::pIDRegClass) {380 visitFunPtrUse(OpReg, SignatureToGReg, GlobalToGReg, MF, MI);381 continue;382 }383 // Skip already processed instructions384 if (MAI.hasRegisterAlias(MF, MO.getReg()))385 continue;386 // Recursively visit dependencies387 if (const MachineInstr *OpDefMI = MRI.getUniqueVRegDef(OpReg)) {388 if (isDeclSection(MRI, *OpDefMI))389 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, *OpDefMI);390 continue;391 }392 // Handle the unexpected case of no unique definition for the SPIR-V393 // instruction394 LLVM_DEBUG({395 dbgs() << "Unexpectedly, no unique definition for the operand ";396 MO.print(dbgs());397 dbgs() << "\nInstruction: ";398 MI.print(dbgs());399 dbgs() << "\n";400 });401 report_fatal_error(402 "No unique definition is found for the virtual register");403 }404 405 MCRegister GReg;406 bool IsFunDef = false;407 if (TII->isSpecConstantInstr(MI)) {408 GReg = MAI.getNextIDRegister();409 MAI.MS[SPIRV::MB_TypeConstVars].push_back(&MI);410 } else if (Opcode == SPIRV::OpFunction ||411 Opcode == SPIRV::OpFunctionParameter) {412 GReg = handleFunctionOrParameter(MF, MI, GlobalToGReg, IsFunDef);413 } else if (Opcode == SPIRV::OpTypeStruct ||414 Opcode == SPIRV::OpConstantComposite) {415 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);416 const MachineInstr *NextInstr = MI.getNextNode();417 while (NextInstr &&418 ((Opcode == SPIRV::OpTypeStruct &&419 NextInstr->getOpcode() == SPIRV::OpTypeStructContinuedINTEL) ||420 (Opcode == SPIRV::OpConstantComposite &&421 NextInstr->getOpcode() ==422 SPIRV::OpConstantCompositeContinuedINTEL))) {423 MCRegister Tmp = handleTypeDeclOrConstant(*NextInstr, SignatureToGReg);424 MAI.setRegisterAlias(MF, NextInstr->getOperand(0).getReg(), Tmp);425 MAI.setSkipEmission(NextInstr);426 NextInstr = NextInstr->getNextNode();427 }428 } else if (TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||429 TII->isInlineAsmDefInstr(MI)) {430 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);431 } else if (Opcode == SPIRV::OpVariable) {432 GReg = handleVariable(MF, MI, GlobalToGReg);433 } else {434 LLVM_DEBUG({435 dbgs() << "\nInstruction: ";436 MI.print(dbgs());437 dbgs() << "\n";438 });439 llvm_unreachable("Unexpected instruction is visited");440 }441 MAI.setRegisterAlias(MF, MI.getOperand(0).getReg(), GReg);442 if (!IsFunDef)443 MAI.setSkipEmission(&MI);444}445 446MCRegister SPIRVModuleAnalysis::handleFunctionOrParameter(447 const MachineFunction *MF, const MachineInstr &MI,448 std::map<const Value *, unsigned> &GlobalToGReg, bool &IsFunDef) {449 const Value *GObj = GR->getGlobalObject(MF, MI.getOperand(0).getReg());450 assert(GObj && "Unregistered global definition");451 const Function *F = dyn_cast<Function>(GObj);452 if (!F)453 F = dyn_cast<Argument>(GObj)->getParent();454 assert(F && "Expected a reference to a function or an argument");455 IsFunDef = !F->isDeclaration();456 auto [It, Inserted] = GlobalToGReg.try_emplace(GObj);457 if (!Inserted)458 return It->second;459 MCRegister GReg = MAI.getNextIDRegister();460 It->second = GReg;461 if (!IsFunDef)462 MAI.MS[SPIRV::MB_ExtFuncDecls].push_back(&MI);463 return GReg;464}465 466MCRegister467SPIRVModuleAnalysis::handleTypeDeclOrConstant(const MachineInstr &MI,468 InstrGRegsMap &SignatureToGReg) {469 InstrSignature MISign = instrToSignature(MI, MAI, false);470 auto [It, Inserted] = SignatureToGReg.try_emplace(MISign);471 if (!Inserted)472 return It->second;473 MCRegister GReg = MAI.getNextIDRegister();474 It->second = GReg;475 MAI.MS[SPIRV::MB_TypeConstVars].push_back(&MI);476 return GReg;477}478 479MCRegister SPIRVModuleAnalysis::handleVariable(480 const MachineFunction *MF, const MachineInstr &MI,481 std::map<const Value *, unsigned> &GlobalToGReg) {482 MAI.GlobalVarList.push_back(&MI);483 const Value *GObj = GR->getGlobalObject(MF, MI.getOperand(0).getReg());484 assert(GObj && "Unregistered global definition");485 auto [It, Inserted] = GlobalToGReg.try_emplace(GObj);486 if (!Inserted)487 return It->second;488 MCRegister GReg = MAI.getNextIDRegister();489 It->second = GReg;490 MAI.MS[SPIRV::MB_TypeConstVars].push_back(&MI);491 return GReg;492}493 494void SPIRVModuleAnalysis::collectDeclarations(const Module &M) {495 InstrGRegsMap SignatureToGReg;496 std::map<const Value *, unsigned> GlobalToGReg;497 for (const Function &F : M) {498 MachineFunction *MF = MMI->getMachineFunction(F);499 if (!MF)500 continue;501 const MachineRegisterInfo &MRI = MF->getRegInfo();502 unsigned PastHeader = 0;503 for (MachineBasicBlock &MBB : *MF) {504 for (MachineInstr &MI : MBB) {505 if (MI.getNumOperands() == 0)506 continue;507 unsigned Opcode = MI.getOpcode();508 if (Opcode == SPIRV::OpFunction) {509 if (PastHeader == 0) {510 PastHeader = 1;511 continue;512 }513 } else if (Opcode == SPIRV::OpFunctionParameter) {514 if (PastHeader < 2)515 continue;516 } else if (PastHeader > 0) {517 PastHeader = 2;518 }519 520 const MachineOperand &DefMO = MI.getOperand(0);521 switch (Opcode) {522 case SPIRV::OpExtension:523 MAI.Reqs.addExtension(SPIRV::Extension::Extension(DefMO.getImm()));524 MAI.setSkipEmission(&MI);525 break;526 case SPIRV::OpCapability:527 MAI.Reqs.addCapability(SPIRV::Capability::Capability(DefMO.getImm()));528 MAI.setSkipEmission(&MI);529 if (PastHeader > 0)530 PastHeader = 2;531 break;532 default:533 if (DefMO.isReg() && isDeclSection(MRI, MI) &&534 !MAI.hasRegisterAlias(MF, DefMO.getReg()))535 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI);536 }537 }538 }539 }540}541 542// Look for IDs declared with Import linkage, and map the corresponding function543// to the register defining that variable (which will usually be the result of544// an OpFunction). This lets us call externally imported functions using545// the correct ID registers.546void SPIRVModuleAnalysis::collectFuncNames(MachineInstr &MI,547 const Function *F) {548 if (MI.getOpcode() == SPIRV::OpDecorate) {549 // If it's got Import linkage.550 auto Dec = MI.getOperand(1).getImm();551 if (Dec == SPIRV::Decoration::LinkageAttributes) {552 auto Lnk = MI.getOperand(MI.getNumOperands() - 1).getImm();553 if (Lnk == SPIRV::LinkageType::Import) {554 // Map imported function name to function ID register.555 const Function *ImportedFunc =556 F->getParent()->getFunction(getStringImm(MI, 2));557 Register Target = MI.getOperand(0).getReg();558 MAI.FuncMap[ImportedFunc] = MAI.getRegisterAlias(MI.getMF(), Target);559 }560 }561 } else if (MI.getOpcode() == SPIRV::OpFunction) {562 // Record all internal OpFunction declarations.563 Register Reg = MI.defs().begin()->getReg();564 MCRegister GlobalReg = MAI.getRegisterAlias(MI.getMF(), Reg);565 assert(GlobalReg.isValid());566 MAI.FuncMap[F] = GlobalReg;567 }568}569 570// Collect the given instruction in the specified MS. We assume global register571// numbering has already occurred by this point. We can directly compare reg572// arguments when detecting duplicates.573static void collectOtherInstr(MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI,574 SPIRV::ModuleSectionType MSType, InstrTraces &IS,575 bool Append = true) {576 MAI.setSkipEmission(&MI);577 InstrSignature MISign = instrToSignature(MI, MAI, true);578 auto FoundMI = IS.insert(std::move(MISign));579 if (!FoundMI.second) {580 if (MI.getOpcode() == SPIRV::OpDecorate) {581 assert(MI.getNumOperands() >= 2 &&582 "Decoration instructions must have at least 2 operands");583 assert(MSType == SPIRV::MB_Annotations &&584 "Only OpDecorate instructions can be duplicates");585 // For FPFastMathMode decoration, we need to merge the flags of the586 // duplicate decoration with the original one, so we need to find the587 // original instruction that has the same signature. For the rest of588 // instructions, we will simply skip the duplicate.589 if (MI.getOperand(1).getImm() != SPIRV::Decoration::FPFastMathMode)590 return; // Skip duplicates of other decorations.591 592 const SPIRV::InstrList &Decorations = MAI.MS[MSType];593 for (const MachineInstr *OrigMI : Decorations) {594 if (instrToSignature(*OrigMI, MAI, true) == MISign) {595 assert(OrigMI->getNumOperands() == MI.getNumOperands() &&596 "Original instruction must have the same number of operands");597 assert(598 OrigMI->getNumOperands() == 3 &&599 "FPFastMathMode decoration must have 3 operands for OpDecorate");600 unsigned OrigFlags = OrigMI->getOperand(2).getImm();601 unsigned NewFlags = MI.getOperand(2).getImm();602 if (OrigFlags == NewFlags)603 return; // No need to merge, the flags are the same.604 605 // Emit warning about possible conflict between flags.606 unsigned FinalFlags = OrigFlags | NewFlags;607 llvm::errs()608 << "Warning: Conflicting FPFastMathMode decoration flags "609 "in instruction: "610 << *OrigMI << "Original flags: " << OrigFlags611 << ", new flags: " << NewFlags612 << ". They will be merged on a best effort basis, but not "613 "validated. Final flags: "614 << FinalFlags << "\n";615 MachineInstr *OrigMINonConst = const_cast<MachineInstr *>(OrigMI);616 MachineOperand &OrigFlagsOp = OrigMINonConst->getOperand(2);617 OrigFlagsOp = MachineOperand::CreateImm(FinalFlags);618 return; // Merge done, so we found a duplicate; don't add it to MAI.MS619 }620 }621 assert(false && "No original instruction found for the duplicate "622 "OpDecorate, but we found one in IS.");623 }624 return; // insert failed, so we found a duplicate; don't add it to MAI.MS625 }626 // No duplicates, so add it.627 if (Append)628 MAI.MS[MSType].push_back(&MI);629 else630 MAI.MS[MSType].insert(MAI.MS[MSType].begin(), &MI);631}632 633// Some global instructions make reference to function-local ID regs, so cannot634// be correctly collected until these registers are globally numbered.635void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {636 InstrTraces IS;637 for (const Function &F : M) {638 if (F.isDeclaration())639 continue;640 MachineFunction *MF = MMI->getMachineFunction(F);641 assert(MF);642 643 for (MachineBasicBlock &MBB : *MF)644 for (MachineInstr &MI : MBB) {645 if (MAI.getSkipEmission(&MI))646 continue;647 const unsigned OpCode = MI.getOpcode();648 if (OpCode == SPIRV::OpString) {649 collectOtherInstr(MI, MAI, SPIRV::MB_DebugStrings, IS);650 } else if (OpCode == SPIRV::OpExtInst && MI.getOperand(2).isImm() &&651 MI.getOperand(2).getImm() ==652 SPIRV::InstructionSet::653 NonSemantic_Shader_DebugInfo_100) {654 MachineOperand Ins = MI.getOperand(3);655 namespace NS = SPIRV::NonSemanticExtInst;656 static constexpr int64_t GlobalNonSemanticDITy[] = {657 NS::DebugSource, NS::DebugCompilationUnit, NS::DebugInfoNone,658 NS::DebugTypeBasic, NS::DebugTypePointer};659 bool IsGlobalDI = false;660 for (unsigned Idx = 0; Idx < std::size(GlobalNonSemanticDITy); ++Idx)661 IsGlobalDI |= Ins.getImm() == GlobalNonSemanticDITy[Idx];662 if (IsGlobalDI)663 collectOtherInstr(MI, MAI, SPIRV::MB_NonSemanticGlobalDI, IS);664 } else if (OpCode == SPIRV::OpName || OpCode == SPIRV::OpMemberName) {665 collectOtherInstr(MI, MAI, SPIRV::MB_DebugNames, IS);666 } else if (OpCode == SPIRV::OpEntryPoint) {667 collectOtherInstr(MI, MAI, SPIRV::MB_EntryPoints, IS);668 } else if (TII->isAliasingInstr(MI)) {669 collectOtherInstr(MI, MAI, SPIRV::MB_AliasingInsts, IS);670 } else if (TII->isDecorationInstr(MI)) {671 collectOtherInstr(MI, MAI, SPIRV::MB_Annotations, IS);672 collectFuncNames(MI, &F);673 } else if (TII->isConstantInstr(MI)) {674 // Now OpSpecConstant*s are not in DT,675 // but they need to be collected anyway.676 collectOtherInstr(MI, MAI, SPIRV::MB_TypeConstVars, IS);677 } else if (OpCode == SPIRV::OpFunction) {678 collectFuncNames(MI, &F);679 } else if (OpCode == SPIRV::OpTypeForwardPointer) {680 collectOtherInstr(MI, MAI, SPIRV::MB_TypeConstVars, IS, false);681 }682 }683 }684}685 686// Number registers in all functions globally from 0 onwards and store687// the result in global register alias table. Some registers are already688// numbered.689void SPIRVModuleAnalysis::numberRegistersGlobally(const Module &M) {690 for (const Function &F : M) {691 if (F.isDeclaration())692 continue;693 MachineFunction *MF = MMI->getMachineFunction(F);694 assert(MF);695 for (MachineBasicBlock &MBB : *MF) {696 for (MachineInstr &MI : MBB) {697 for (MachineOperand &Op : MI.operands()) {698 if (!Op.isReg())699 continue;700 Register Reg = Op.getReg();701 if (MAI.hasRegisterAlias(MF, Reg))702 continue;703 MCRegister NewReg = MAI.getNextIDRegister();704 MAI.setRegisterAlias(MF, Reg, NewReg);705 }706 if (MI.getOpcode() != SPIRV::OpExtInst)707 continue;708 auto Set = MI.getOperand(2).getImm();709 auto [It, Inserted] = MAI.ExtInstSetMap.try_emplace(Set);710 if (Inserted)711 It->second = MAI.getNextIDRegister();712 }713 }714 }715}716 717// RequirementHandler implementations.718void SPIRV::RequirementHandler::getAndAddRequirements(719 SPIRV::OperandCategory::OperandCategory Category, uint32_t i,720 const SPIRVSubtarget &ST) {721 addRequirements(getSymbolicOperandRequirements(Category, i, ST, *this));722}723 724void SPIRV::RequirementHandler::recursiveAddCapabilities(725 const CapabilityList &ToPrune) {726 for (const auto &Cap : ToPrune) {727 AllCaps.insert(Cap);728 CapabilityList ImplicitDecls =729 getSymbolicOperandCapabilities(OperandCategory::CapabilityOperand, Cap);730 recursiveAddCapabilities(ImplicitDecls);731 }732}733 734void SPIRV::RequirementHandler::addCapabilities(const CapabilityList &ToAdd) {735 for (const auto &Cap : ToAdd) {736 bool IsNewlyInserted = AllCaps.insert(Cap).second;737 if (!IsNewlyInserted) // Don't re-add if it's already been declared.738 continue;739 CapabilityList ImplicitDecls =740 getSymbolicOperandCapabilities(OperandCategory::CapabilityOperand, Cap);741 recursiveAddCapabilities(ImplicitDecls);742 MinimalCaps.push_back(Cap);743 }744}745 746void SPIRV::RequirementHandler::addRequirements(747 const SPIRV::Requirements &Req) {748 if (!Req.IsSatisfiable)749 report_fatal_error("Adding SPIR-V requirements this target can't satisfy.");750 751 if (Req.Cap.has_value())752 addCapabilities({Req.Cap.value()});753 754 addExtensions(Req.Exts);755 756 if (!Req.MinVer.empty()) {757 if (!MaxVersion.empty() && Req.MinVer > MaxVersion) {758 LLVM_DEBUG(dbgs() << "Conflicting version requirements: >= " << Req.MinVer759 << " and <= " << MaxVersion << "\n");760 report_fatal_error("Adding SPIR-V requirements that can't be satisfied.");761 }762 763 if (MinVersion.empty() || Req.MinVer > MinVersion)764 MinVersion = Req.MinVer;765 }766 767 if (!Req.MaxVer.empty()) {768 if (!MinVersion.empty() && Req.MaxVer < MinVersion) {769 LLVM_DEBUG(dbgs() << "Conflicting version requirements: <= " << Req.MaxVer770 << " and >= " << MinVersion << "\n");771 report_fatal_error("Adding SPIR-V requirements that can't be satisfied.");772 }773 774 if (MaxVersion.empty() || Req.MaxVer < MaxVersion)775 MaxVersion = Req.MaxVer;776 }777}778 779void SPIRV::RequirementHandler::checkSatisfiable(780 const SPIRVSubtarget &ST) const {781 // Report as many errors as possible before aborting the compilation.782 bool IsSatisfiable = true;783 auto TargetVer = ST.getSPIRVVersion();784 785 if (!MaxVersion.empty() && !TargetVer.empty() && MaxVersion < TargetVer) {786 LLVM_DEBUG(787 dbgs() << "Target SPIR-V version too high for required features\n"788 << "Required max version: " << MaxVersion << " target version "789 << TargetVer << "\n");790 IsSatisfiable = false;791 }792 793 if (!MinVersion.empty() && !TargetVer.empty() && MinVersion > TargetVer) {794 LLVM_DEBUG(dbgs() << "Target SPIR-V version too low for required features\n"795 << "Required min version: " << MinVersion796 << " target version " << TargetVer << "\n");797 IsSatisfiable = false;798 }799 800 if (!MinVersion.empty() && !MaxVersion.empty() && MinVersion > MaxVersion) {801 LLVM_DEBUG(802 dbgs()803 << "Version is too low for some features and too high for others.\n"804 << "Required SPIR-V min version: " << MinVersion805 << " required SPIR-V max version " << MaxVersion << "\n");806 IsSatisfiable = false;807 }808 809 AvoidCapabilitiesSet AvoidCaps;810 if (!ST.isShader())811 AvoidCaps.S.insert(SPIRV::Capability::Shader);812 else813 AvoidCaps.S.insert(SPIRV::Capability::Kernel);814 815 for (auto Cap : MinimalCaps) {816 if (AvailableCaps.contains(Cap) && !AvoidCaps.S.contains(Cap))817 continue;818 LLVM_DEBUG(dbgs() << "Capability not supported: "819 << getSymbolicOperandMnemonic(820 OperandCategory::CapabilityOperand, Cap)821 << "\n");822 IsSatisfiable = false;823 }824 825 for (auto Ext : AllExtensions) {826 if (ST.canUseExtension(Ext))827 continue;828 LLVM_DEBUG(dbgs() << "Extension not supported: "829 << getSymbolicOperandMnemonic(830 OperandCategory::ExtensionOperand, Ext)831 << "\n");832 IsSatisfiable = false;833 }834 835 if (!IsSatisfiable)836 report_fatal_error("Unable to meet SPIR-V requirements for this target.");837}838 839// Add the given capabilities and all their implicitly defined capabilities too.840void SPIRV::RequirementHandler::addAvailableCaps(const CapabilityList &ToAdd) {841 for (const auto Cap : ToAdd)842 if (AvailableCaps.insert(Cap).second)843 addAvailableCaps(getSymbolicOperandCapabilities(844 SPIRV::OperandCategory::CapabilityOperand, Cap));845}846 847void SPIRV::RequirementHandler::removeCapabilityIf(848 const Capability::Capability ToRemove,849 const Capability::Capability IfPresent) {850 if (AllCaps.contains(IfPresent))851 AllCaps.erase(ToRemove);852}853 854namespace llvm {855namespace SPIRV {856void RequirementHandler::initAvailableCapabilities(const SPIRVSubtarget &ST) {857 // Provided by both all supported Vulkan versions and OpenCl.858 addAvailableCaps({Capability::Shader, Capability::Linkage, Capability::Int8,859 Capability::Int16});860 861 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 3)))862 addAvailableCaps({Capability::GroupNonUniform,863 Capability::GroupNonUniformVote,864 Capability::GroupNonUniformArithmetic,865 Capability::GroupNonUniformBallot,866 Capability::GroupNonUniformClustered,867 Capability::GroupNonUniformShuffle,868 Capability::GroupNonUniformShuffleRelative});869 870 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 6)))871 addAvailableCaps({Capability::DotProduct, Capability::DotProductInputAll,872 Capability::DotProductInput4x8Bit,873 Capability::DotProductInput4x8BitPacked,874 Capability::DemoteToHelperInvocation});875 876 // Add capabilities enabled by extensions.877 for (auto Extension : ST.getAllAvailableExtensions()) {878 CapabilityList EnabledCapabilities =879 getCapabilitiesEnabledByExtension(Extension);880 addAvailableCaps(EnabledCapabilities);881 }882 883 if (!ST.isShader()) {884 initAvailableCapabilitiesForOpenCL(ST);885 return;886 }887 888 if (ST.isShader()) {889 initAvailableCapabilitiesForVulkan(ST);890 return;891 }892 893 report_fatal_error("Unimplemented environment for SPIR-V generation.");894}895 896void RequirementHandler::initAvailableCapabilitiesForOpenCL(897 const SPIRVSubtarget &ST) {898 // Add the min requirements for different OpenCL and SPIR-V versions.899 addAvailableCaps({Capability::Addresses, Capability::Float16Buffer,900 Capability::Kernel, Capability::Vector16,901 Capability::Groups, Capability::GenericPointer,902 Capability::StorageImageWriteWithoutFormat,903 Capability::StorageImageReadWithoutFormat});904 if (ST.hasOpenCLFullProfile())905 addAvailableCaps({Capability::Int64, Capability::Int64Atomics});906 if (ST.hasOpenCLImageSupport()) {907 addAvailableCaps({Capability::ImageBasic, Capability::LiteralSampler,908 Capability::Image1D, Capability::SampledBuffer,909 Capability::ImageBuffer});910 if (ST.isAtLeastOpenCLVer(VersionTuple(2, 0)))911 addAvailableCaps({Capability::ImageReadWrite});912 }913 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 1)) &&914 ST.isAtLeastOpenCLVer(VersionTuple(2, 2)))915 addAvailableCaps({Capability::SubgroupDispatch, Capability::PipeStorage});916 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 4)))917 addAvailableCaps({Capability::DenormPreserve, Capability::DenormFlushToZero,918 Capability::SignedZeroInfNanPreserve,919 Capability::RoundingModeRTE,920 Capability::RoundingModeRTZ});921 // TODO: verify if this needs some checks.922 addAvailableCaps({Capability::Float16, Capability::Float64});923 924 // TODO: add OpenCL extensions.925}926 927void RequirementHandler::initAvailableCapabilitiesForVulkan(928 const SPIRVSubtarget &ST) {929 930 // Core in Vulkan 1.1 and earlier.931 addAvailableCaps({Capability::Int64, Capability::Float16, Capability::Float64,932 Capability::GroupNonUniform, Capability::Image1D,933 Capability::SampledBuffer, Capability::ImageBuffer,934 Capability::UniformBufferArrayDynamicIndexing,935 Capability::SampledImageArrayDynamicIndexing,936 Capability::StorageBufferArrayDynamicIndexing,937 Capability::StorageImageArrayDynamicIndexing,938 Capability::DerivativeControl});939 940 // Became core in Vulkan 1.2941 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 5))) {942 addAvailableCaps(943 {Capability::ShaderNonUniformEXT, Capability::RuntimeDescriptorArrayEXT,944 Capability::InputAttachmentArrayDynamicIndexingEXT,945 Capability::UniformTexelBufferArrayDynamicIndexingEXT,946 Capability::StorageTexelBufferArrayDynamicIndexingEXT,947 Capability::UniformBufferArrayNonUniformIndexingEXT,948 Capability::SampledImageArrayNonUniformIndexingEXT,949 Capability::StorageBufferArrayNonUniformIndexingEXT,950 Capability::StorageImageArrayNonUniformIndexingEXT,951 Capability::InputAttachmentArrayNonUniformIndexingEXT,952 Capability::UniformTexelBufferArrayNonUniformIndexingEXT,953 Capability::StorageTexelBufferArrayNonUniformIndexingEXT});954 }955 956 // Became core in Vulkan 1.3957 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 6)))958 addAvailableCaps({Capability::StorageImageWriteWithoutFormat,959 Capability::StorageImageReadWithoutFormat});960}961 962} // namespace SPIRV963} // namespace llvm964 965// Add the required capabilities from a decoration instruction (including966// BuiltIns).967static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex,968 SPIRV::RequirementHandler &Reqs,969 const SPIRVSubtarget &ST) {970 int64_t DecOp = MI.getOperand(DecIndex).getImm();971 auto Dec = static_cast<SPIRV::Decoration::Decoration>(DecOp);972 Reqs.addRequirements(getSymbolicOperandRequirements(973 SPIRV::OperandCategory::DecorationOperand, Dec, ST, Reqs));974 975 if (Dec == SPIRV::Decoration::BuiltIn) {976 int64_t BuiltInOp = MI.getOperand(DecIndex + 1).getImm();977 auto BuiltIn = static_cast<SPIRV::BuiltIn::BuiltIn>(BuiltInOp);978 Reqs.addRequirements(getSymbolicOperandRequirements(979 SPIRV::OperandCategory::BuiltInOperand, BuiltIn, ST, Reqs));980 } else if (Dec == SPIRV::Decoration::LinkageAttributes) {981 int64_t LinkageOp = MI.getOperand(MI.getNumOperands() - 1).getImm();982 SPIRV::LinkageType::LinkageType LnkType =983 static_cast<SPIRV::LinkageType::LinkageType>(LinkageOp);984 if (LnkType == SPIRV::LinkageType::LinkOnceODR)985 Reqs.addExtension(SPIRV::Extension::SPV_KHR_linkonce_odr);986 } else if (Dec == SPIRV::Decoration::CacheControlLoadINTEL ||987 Dec == SPIRV::Decoration::CacheControlStoreINTEL) {988 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_cache_controls);989 } else if (Dec == SPIRV::Decoration::HostAccessINTEL) {990 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_global_variable_host_access);991 } else if (Dec == SPIRV::Decoration::InitModeINTEL ||992 Dec == SPIRV::Decoration::ImplementInRegisterMapINTEL) {993 Reqs.addExtension(994 SPIRV::Extension::SPV_INTEL_global_variable_fpga_decorations);995 } else if (Dec == SPIRV::Decoration::NonUniformEXT) {996 Reqs.addRequirements(SPIRV::Capability::ShaderNonUniformEXT);997 } else if (Dec == SPIRV::Decoration::FPMaxErrorDecorationINTEL) {998 Reqs.addRequirements(SPIRV::Capability::FPMaxErrorINTEL);999 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_fp_max_error);1000 } else if (Dec == SPIRV::Decoration::FPFastMathMode) {1001 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2)) {1002 Reqs.addRequirements(SPIRV::Capability::FloatControls2);1003 Reqs.addExtension(SPIRV::Extension::SPV_KHR_float_controls2);1004 }1005 }1006}1007 1008// Add requirements for image handling.1009static void addOpTypeImageReqs(const MachineInstr &MI,1010 SPIRV::RequirementHandler &Reqs,1011 const SPIRVSubtarget &ST) {1012 assert(MI.getNumOperands() >= 8 && "Insufficient operands for OpTypeImage");1013 // The operand indices used here are based on the OpTypeImage layout, which1014 // the MachineInstr follows as well.1015 int64_t ImgFormatOp = MI.getOperand(7).getImm();1016 auto ImgFormat = static_cast<SPIRV::ImageFormat::ImageFormat>(ImgFormatOp);1017 Reqs.getAndAddRequirements(SPIRV::OperandCategory::ImageFormatOperand,1018 ImgFormat, ST);1019 1020 bool IsArrayed = MI.getOperand(4).getImm() == 1;1021 bool IsMultisampled = MI.getOperand(5).getImm() == 1;1022 bool NoSampler = MI.getOperand(6).getImm() == 2;1023 // Add dimension requirements.1024 assert(MI.getOperand(2).isImm());1025 switch (MI.getOperand(2).getImm()) {1026 case SPIRV::Dim::DIM_1D:1027 Reqs.addRequirements(NoSampler ? SPIRV::Capability::Image1D1028 : SPIRV::Capability::Sampled1D);1029 break;1030 case SPIRV::Dim::DIM_2D:1031 if (IsMultisampled && NoSampler)1032 Reqs.addRequirements(SPIRV::Capability::ImageMSArray);1033 break;1034 case SPIRV::Dim::DIM_Cube:1035 Reqs.addRequirements(SPIRV::Capability::Shader);1036 if (IsArrayed)1037 Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageCubeArray1038 : SPIRV::Capability::SampledCubeArray);1039 break;1040 case SPIRV::Dim::DIM_Rect:1041 Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageRect1042 : SPIRV::Capability::SampledRect);1043 break;1044 case SPIRV::Dim::DIM_Buffer:1045 Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageBuffer1046 : SPIRV::Capability::SampledBuffer);1047 break;1048 case SPIRV::Dim::DIM_SubpassData:1049 Reqs.addRequirements(SPIRV::Capability::InputAttachment);1050 break;1051 }1052 1053 // Has optional access qualifier.1054 if (!ST.isShader()) {1055 if (MI.getNumOperands() > 8 &&1056 MI.getOperand(8).getImm() == SPIRV::AccessQualifier::ReadWrite)1057 Reqs.addRequirements(SPIRV::Capability::ImageReadWrite);1058 else1059 Reqs.addRequirements(SPIRV::Capability::ImageBasic);1060 }1061}1062 1063static bool isBFloat16Type(const SPIRVType *TypeDef) {1064 return TypeDef && TypeDef->getNumOperands() == 3 &&1065 TypeDef->getOpcode() == SPIRV::OpTypeFloat &&1066 TypeDef->getOperand(1).getImm() == 16 &&1067 TypeDef->getOperand(2).getImm() == SPIRV::FPEncoding::BFloat16KHR;1068}1069 1070// Add requirements for handling atomic float instructions1071#define ATOM_FLT_REQ_EXT_MSG(ExtName) \1072 "The atomic float instruction requires the following SPIR-V " \1073 "extension: SPV_EXT_shader_atomic_float" ExtName1074static void AddAtomicFloatRequirements(const MachineInstr &MI,1075 SPIRV::RequirementHandler &Reqs,1076 const SPIRVSubtarget &ST) {1077 assert(MI.getOperand(1).isReg() &&1078 "Expect register operand in atomic float instruction");1079 Register TypeReg = MI.getOperand(1).getReg();1080 SPIRVType *TypeDef = MI.getMF()->getRegInfo().getVRegDef(TypeReg);1081 if (TypeDef->getOpcode() != SPIRV::OpTypeFloat)1082 report_fatal_error("Result type of an atomic float instruction must be a "1083 "floating-point type scalar");1084 1085 unsigned BitWidth = TypeDef->getOperand(1).getImm();1086 unsigned Op = MI.getOpcode();1087 if (Op == SPIRV::OpAtomicFAddEXT) {1088 if (!ST.canUseExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float_add))1089 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_add"), false);1090 Reqs.addExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float_add);1091 switch (BitWidth) {1092 case 16:1093 if (isBFloat16Type(TypeDef)) {1094 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics))1095 report_fatal_error(1096 "The atomic bfloat16 instruction requires the following SPIR-V "1097 "extension: SPV_INTEL_16bit_atomics",1098 false);1099 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics);1100 Reqs.addCapability(SPIRV::Capability::AtomicBFloat16AddINTEL);1101 } else {1102 if (!ST.canUseExtension(1103 SPIRV::Extension::SPV_EXT_shader_atomic_float16_add))1104 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("16_add"), false);1105 Reqs.addExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float16_add);1106 Reqs.addCapability(SPIRV::Capability::AtomicFloat16AddEXT);1107 }1108 break;1109 case 32:1110 Reqs.addCapability(SPIRV::Capability::AtomicFloat32AddEXT);1111 break;1112 case 64:1113 Reqs.addCapability(SPIRV::Capability::AtomicFloat64AddEXT);1114 break;1115 default:1116 report_fatal_error(1117 "Unexpected floating-point type width in atomic float instruction");1118 }1119 } else {1120 if (!ST.canUseExtension(1121 SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max))1122 report_fatal_error(ATOM_FLT_REQ_EXT_MSG("_min_max"), false);1123 Reqs.addExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max);1124 switch (BitWidth) {1125 case 16:1126 if (isBFloat16Type(TypeDef)) {1127 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics))1128 report_fatal_error(1129 "The atomic bfloat16 instruction requires the following SPIR-V "1130 "extension: SPV_INTEL_16bit_atomics",1131 false);1132 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics);1133 Reqs.addCapability(SPIRV::Capability::AtomicBFloat16MinMaxINTEL);1134 } else {1135 Reqs.addCapability(SPIRV::Capability::AtomicFloat16MinMaxEXT);1136 }1137 break;1138 case 32:1139 Reqs.addCapability(SPIRV::Capability::AtomicFloat32MinMaxEXT);1140 break;1141 case 64:1142 Reqs.addCapability(SPIRV::Capability::AtomicFloat64MinMaxEXT);1143 break;1144 default:1145 report_fatal_error(1146 "Unexpected floating-point type width in atomic float instruction");1147 }1148 }1149}1150 1151bool isUniformTexelBuffer(MachineInstr *ImageInst) {1152 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)1153 return false;1154 uint32_t Dim = ImageInst->getOperand(2).getImm();1155 uint32_t Sampled = ImageInst->getOperand(6).getImm();1156 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 1;1157}1158 1159bool isStorageTexelBuffer(MachineInstr *ImageInst) {1160 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)1161 return false;1162 uint32_t Dim = ImageInst->getOperand(2).getImm();1163 uint32_t Sampled = ImageInst->getOperand(6).getImm();1164 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 2;1165}1166 1167bool isSampledImage(MachineInstr *ImageInst) {1168 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)1169 return false;1170 uint32_t Dim = ImageInst->getOperand(2).getImm();1171 uint32_t Sampled = ImageInst->getOperand(6).getImm();1172 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 1;1173}1174 1175bool isInputAttachment(MachineInstr *ImageInst) {1176 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)1177 return false;1178 uint32_t Dim = ImageInst->getOperand(2).getImm();1179 uint32_t Sampled = ImageInst->getOperand(6).getImm();1180 return Dim == SPIRV::Dim::DIM_SubpassData && Sampled == 2;1181}1182 1183bool isStorageImage(MachineInstr *ImageInst) {1184 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)1185 return false;1186 uint32_t Dim = ImageInst->getOperand(2).getImm();1187 uint32_t Sampled = ImageInst->getOperand(6).getImm();1188 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 2;1189}1190 1191bool isCombinedImageSampler(MachineInstr *SampledImageInst) {1192 if (SampledImageInst->getOpcode() != SPIRV::OpTypeSampledImage)1193 return false;1194 1195 const MachineRegisterInfo &MRI = SampledImageInst->getMF()->getRegInfo();1196 Register ImageReg = SampledImageInst->getOperand(1).getReg();1197 auto *ImageInst = MRI.getUniqueVRegDef(ImageReg);1198 return isSampledImage(ImageInst);1199}1200 1201bool hasNonUniformDecoration(Register Reg, const MachineRegisterInfo &MRI) {1202 for (const auto &MI : MRI.reg_instructions(Reg)) {1203 if (MI.getOpcode() != SPIRV::OpDecorate)1204 continue;1205 1206 uint32_t Dec = MI.getOperand(1).getImm();1207 if (Dec == SPIRV::Decoration::NonUniformEXT)1208 return true;1209 }1210 return false;1211}1212 1213void addOpAccessChainReqs(const MachineInstr &Instr,1214 SPIRV::RequirementHandler &Handler,1215 const SPIRVSubtarget &Subtarget) {1216 const MachineRegisterInfo &MRI = Instr.getMF()->getRegInfo();1217 // Get the result type. If it is an image type, then the shader uses1218 // descriptor indexing. The appropriate capabilities will be added based1219 // on the specifics of the image.1220 Register ResTypeReg = Instr.getOperand(1).getReg();1221 MachineInstr *ResTypeInst = MRI.getUniqueVRegDef(ResTypeReg);1222 1223 assert(ResTypeInst->getOpcode() == SPIRV::OpTypePointer);1224 uint32_t StorageClass = ResTypeInst->getOperand(1).getImm();1225 if (StorageClass != SPIRV::StorageClass::StorageClass::UniformConstant &&1226 StorageClass != SPIRV::StorageClass::StorageClass::Uniform &&1227 StorageClass != SPIRV::StorageClass::StorageClass::StorageBuffer) {1228 return;1229 }1230 1231 bool IsNonUniform =1232 hasNonUniformDecoration(Instr.getOperand(0).getReg(), MRI);1233 1234 auto FirstIndexReg = Instr.getOperand(3).getReg();1235 bool FirstIndexIsConstant =1236 Subtarget.getInstrInfo()->isConstantInstr(*MRI.getVRegDef(FirstIndexReg));1237 1238 if (StorageClass == SPIRV::StorageClass::StorageClass::StorageBuffer) {1239 if (IsNonUniform)1240 Handler.addRequirements(1241 SPIRV::Capability::StorageBufferArrayNonUniformIndexingEXT);1242 else if (!FirstIndexIsConstant)1243 Handler.addRequirements(1244 SPIRV::Capability::StorageBufferArrayDynamicIndexing);1245 return;1246 }1247 1248 Register PointeeTypeReg = ResTypeInst->getOperand(2).getReg();1249 MachineInstr *PointeeType = MRI.getUniqueVRegDef(PointeeTypeReg);1250 if (PointeeType->getOpcode() != SPIRV::OpTypeImage &&1251 PointeeType->getOpcode() != SPIRV::OpTypeSampledImage &&1252 PointeeType->getOpcode() != SPIRV::OpTypeSampler) {1253 return;1254 }1255 1256 if (isUniformTexelBuffer(PointeeType)) {1257 if (IsNonUniform)1258 Handler.addRequirements(1259 SPIRV::Capability::UniformTexelBufferArrayNonUniformIndexingEXT);1260 else if (!FirstIndexIsConstant)1261 Handler.addRequirements(1262 SPIRV::Capability::UniformTexelBufferArrayDynamicIndexingEXT);1263 } else if (isInputAttachment(PointeeType)) {1264 if (IsNonUniform)1265 Handler.addRequirements(1266 SPIRV::Capability::InputAttachmentArrayNonUniformIndexingEXT);1267 else if (!FirstIndexIsConstant)1268 Handler.addRequirements(1269 SPIRV::Capability::InputAttachmentArrayDynamicIndexingEXT);1270 } else if (isStorageTexelBuffer(PointeeType)) {1271 if (IsNonUniform)1272 Handler.addRequirements(1273 SPIRV::Capability::StorageTexelBufferArrayNonUniformIndexingEXT);1274 else if (!FirstIndexIsConstant)1275 Handler.addRequirements(1276 SPIRV::Capability::StorageTexelBufferArrayDynamicIndexingEXT);1277 } else if (isSampledImage(PointeeType) ||1278 isCombinedImageSampler(PointeeType) ||1279 PointeeType->getOpcode() == SPIRV::OpTypeSampler) {1280 if (IsNonUniform)1281 Handler.addRequirements(1282 SPIRV::Capability::SampledImageArrayNonUniformIndexingEXT);1283 else if (!FirstIndexIsConstant)1284 Handler.addRequirements(1285 SPIRV::Capability::SampledImageArrayDynamicIndexing);1286 } else if (isStorageImage(PointeeType)) {1287 if (IsNonUniform)1288 Handler.addRequirements(1289 SPIRV::Capability::StorageImageArrayNonUniformIndexingEXT);1290 else if (!FirstIndexIsConstant)1291 Handler.addRequirements(1292 SPIRV::Capability::StorageImageArrayDynamicIndexing);1293 }1294}1295 1296static bool isImageTypeWithUnknownFormat(SPIRVType *TypeInst) {1297 if (TypeInst->getOpcode() != SPIRV::OpTypeImage)1298 return false;1299 assert(TypeInst->getOperand(7).isImm() && "The image format must be an imm.");1300 return TypeInst->getOperand(7).getImm() == 0;1301}1302 1303static void AddDotProductRequirements(const MachineInstr &MI,1304 SPIRV::RequirementHandler &Reqs,1305 const SPIRVSubtarget &ST) {1306 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product))1307 Reqs.addExtension(SPIRV::Extension::SPV_KHR_integer_dot_product);1308 Reqs.addCapability(SPIRV::Capability::DotProduct);1309 1310 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();1311 assert(MI.getOperand(2).isReg() && "Unexpected operand in dot");1312 // We do not consider what the previous instruction is. This is just used1313 // to get the input register and to check the type.1314 const MachineInstr *Input = MRI.getVRegDef(MI.getOperand(2).getReg());1315 assert(Input->getOperand(1).isReg() && "Unexpected operand in dot input");1316 Register InputReg = Input->getOperand(1).getReg();1317 1318 SPIRVType *TypeDef = MRI.getVRegDef(InputReg);1319 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {1320 assert(TypeDef->getOperand(1).getImm() == 32);1321 Reqs.addCapability(SPIRV::Capability::DotProductInput4x8BitPacked);1322 } else if (TypeDef->getOpcode() == SPIRV::OpTypeVector) {1323 SPIRVType *ScalarTypeDef = MRI.getVRegDef(TypeDef->getOperand(1).getReg());1324 assert(ScalarTypeDef->getOpcode() == SPIRV::OpTypeInt);1325 if (ScalarTypeDef->getOperand(1).getImm() == 8) {1326 assert(TypeDef->getOperand(2).getImm() == 4 &&1327 "Dot operand of 8-bit integer type requires 4 components");1328 Reqs.addCapability(SPIRV::Capability::DotProductInput4x8Bit);1329 } else {1330 Reqs.addCapability(SPIRV::Capability::DotProductInputAll);1331 }1332 }1333}1334 1335void addPrintfRequirements(const MachineInstr &MI,1336 SPIRV::RequirementHandler &Reqs,1337 const SPIRVSubtarget &ST) {1338 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();1339 const SPIRVType *PtrType = GR->getSPIRVTypeForVReg(MI.getOperand(4).getReg());1340 if (PtrType) {1341 MachineOperand ASOp = PtrType->getOperand(1);1342 if (ASOp.isImm()) {1343 unsigned AddrSpace = ASOp.getImm();1344 if (AddrSpace != SPIRV::StorageClass::UniformConstant) {1345 if (!ST.canUseExtension(1346 SPIRV::Extension::1347 SPV_EXT_relaxed_printf_string_address_space)) {1348 report_fatal_error("SPV_EXT_relaxed_printf_string_address_space is "1349 "required because printf uses a format string not "1350 "in constant address space.",1351 false);1352 }1353 Reqs.addExtension(1354 SPIRV::Extension::SPV_EXT_relaxed_printf_string_address_space);1355 }1356 }1357 }1358}1359 1360void addInstrRequirements(const MachineInstr &MI,1361 SPIRV::ModuleAnalysisInfo &MAI,1362 const SPIRVSubtarget &ST) {1363 SPIRV::RequirementHandler &Reqs = MAI.Reqs;1364 switch (MI.getOpcode()) {1365 case SPIRV::OpMemoryModel: {1366 int64_t Addr = MI.getOperand(0).getImm();1367 Reqs.getAndAddRequirements(SPIRV::OperandCategory::AddressingModelOperand,1368 Addr, ST);1369 int64_t Mem = MI.getOperand(1).getImm();1370 Reqs.getAndAddRequirements(SPIRV::OperandCategory::MemoryModelOperand, Mem,1371 ST);1372 break;1373 }1374 case SPIRV::OpEntryPoint: {1375 int64_t Exe = MI.getOperand(0).getImm();1376 Reqs.getAndAddRequirements(SPIRV::OperandCategory::ExecutionModelOperand,1377 Exe, ST);1378 break;1379 }1380 case SPIRV::OpExecutionMode:1381 case SPIRV::OpExecutionModeId: {1382 int64_t Exe = MI.getOperand(1).getImm();1383 Reqs.getAndAddRequirements(SPIRV::OperandCategory::ExecutionModeOperand,1384 Exe, ST);1385 break;1386 }1387 case SPIRV::OpTypeMatrix:1388 Reqs.addCapability(SPIRV::Capability::Matrix);1389 break;1390 case SPIRV::OpTypeInt: {1391 unsigned BitWidth = MI.getOperand(1).getImm();1392 if (BitWidth == 64)1393 Reqs.addCapability(SPIRV::Capability::Int64);1394 else if (BitWidth == 16)1395 Reqs.addCapability(SPIRV::Capability::Int16);1396 else if (BitWidth == 8)1397 Reqs.addCapability(SPIRV::Capability::Int8);1398 break;1399 }1400 case SPIRV::OpDot: {1401 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();1402 SPIRVType *TypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());1403 if (isBFloat16Type(TypeDef))1404 Reqs.addCapability(SPIRV::Capability::BFloat16DotProductKHR);1405 break;1406 }1407 case SPIRV::OpTypeFloat: {1408 unsigned BitWidth = MI.getOperand(1).getImm();1409 if (BitWidth == 64)1410 Reqs.addCapability(SPIRV::Capability::Float64);1411 else if (BitWidth == 16) {1412 if (isBFloat16Type(&MI)) {1413 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_bfloat16))1414 report_fatal_error("OpTypeFloat type with bfloat requires the "1415 "following SPIR-V extension: SPV_KHR_bfloat16",1416 false);1417 Reqs.addExtension(SPIRV::Extension::SPV_KHR_bfloat16);1418 Reqs.addCapability(SPIRV::Capability::BFloat16TypeKHR);1419 } else {1420 Reqs.addCapability(SPIRV::Capability::Float16);1421 }1422 }1423 break;1424 }1425 case SPIRV::OpTypeVector: {1426 unsigned NumComponents = MI.getOperand(2).getImm();1427 if (NumComponents == 8 || NumComponents == 16)1428 Reqs.addCapability(SPIRV::Capability::Vector16);1429 break;1430 }1431 case SPIRV::OpTypePointer: {1432 auto SC = MI.getOperand(1).getImm();1433 Reqs.getAndAddRequirements(SPIRV::OperandCategory::StorageClassOperand, SC,1434 ST);1435 // If it's a type of pointer to float16 targeting OpenCL, add Float16Buffer1436 // capability.1437 if (ST.isShader())1438 break;1439 assert(MI.getOperand(2).isReg());1440 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();1441 SPIRVType *TypeDef = MRI.getVRegDef(MI.getOperand(2).getReg());1442 if ((TypeDef->getNumOperands() == 2) &&1443 (TypeDef->getOpcode() == SPIRV::OpTypeFloat) &&1444 (TypeDef->getOperand(1).getImm() == 16))1445 Reqs.addCapability(SPIRV::Capability::Float16Buffer);1446 break;1447 }1448 case SPIRV::OpExtInst: {1449 if (MI.getOperand(2).getImm() ==1450 static_cast<int64_t>(1451 SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100)) {1452 Reqs.addExtension(SPIRV::Extension::SPV_KHR_non_semantic_info);1453 break;1454 }1455 if (MI.getOperand(3).getImm() ==1456 static_cast<int64_t>(SPIRV::OpenCLExtInst::printf)) {1457 addPrintfRequirements(MI, Reqs, ST);1458 break;1459 }1460 // TODO: handle bfloat16 extended instructions when1461 // SPV_INTEL_bfloat16_arithmetic is enabled.1462 break;1463 }1464 case SPIRV::OpAliasDomainDeclINTEL:1465 case SPIRV::OpAliasScopeDeclINTEL:1466 case SPIRV::OpAliasScopeListDeclINTEL: {1467 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing);1468 Reqs.addCapability(SPIRV::Capability::MemoryAccessAliasingINTEL);1469 break;1470 }1471 case SPIRV::OpBitReverse:1472 case SPIRV::OpBitFieldInsert:1473 case SPIRV::OpBitFieldSExtract:1474 case SPIRV::OpBitFieldUExtract:1475 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions)) {1476 Reqs.addCapability(SPIRV::Capability::Shader);1477 break;1478 }1479 Reqs.addExtension(SPIRV::Extension::SPV_KHR_bit_instructions);1480 Reqs.addCapability(SPIRV::Capability::BitInstructions);1481 break;1482 case SPIRV::OpTypeRuntimeArray:1483 Reqs.addCapability(SPIRV::Capability::Shader);1484 break;1485 case SPIRV::OpTypeOpaque:1486 case SPIRV::OpTypeEvent:1487 Reqs.addCapability(SPIRV::Capability::Kernel);1488 break;1489 case SPIRV::OpTypePipe:1490 case SPIRV::OpTypeReserveId:1491 Reqs.addCapability(SPIRV::Capability::Pipes);1492 break;1493 case SPIRV::OpTypeDeviceEvent:1494 case SPIRV::OpTypeQueue:1495 case SPIRV::OpBuildNDRange:1496 Reqs.addCapability(SPIRV::Capability::DeviceEnqueue);1497 break;1498 case SPIRV::OpDecorate:1499 case SPIRV::OpDecorateId:1500 case SPIRV::OpDecorateString:1501 addOpDecorateReqs(MI, 1, Reqs, ST);1502 break;1503 case SPIRV::OpMemberDecorate:1504 case SPIRV::OpMemberDecorateString:1505 addOpDecorateReqs(MI, 2, Reqs, ST);1506 break;1507 case SPIRV::OpInBoundsPtrAccessChain:1508 Reqs.addCapability(SPIRV::Capability::Addresses);1509 break;1510 case SPIRV::OpConstantSampler:1511 Reqs.addCapability(SPIRV::Capability::LiteralSampler);1512 break;1513 case SPIRV::OpInBoundsAccessChain:1514 case SPIRV::OpAccessChain:1515 addOpAccessChainReqs(MI, Reqs, ST);1516 break;1517 case SPIRV::OpTypeImage:1518 addOpTypeImageReqs(MI, Reqs, ST);1519 break;1520 case SPIRV::OpTypeSampler:1521 if (!ST.isShader()) {1522 Reqs.addCapability(SPIRV::Capability::ImageBasic);1523 }1524 break;1525 case SPIRV::OpTypeForwardPointer:1526 // TODO: check if it's OpenCL's kernel.1527 Reqs.addCapability(SPIRV::Capability::Addresses);1528 break;1529 case SPIRV::OpAtomicFlagTestAndSet:1530 case SPIRV::OpAtomicLoad:1531 case SPIRV::OpAtomicStore:1532 case SPIRV::OpAtomicExchange:1533 case SPIRV::OpAtomicCompareExchange:1534 case SPIRV::OpAtomicIIncrement:1535 case SPIRV::OpAtomicIDecrement:1536 case SPIRV::OpAtomicIAdd:1537 case SPIRV::OpAtomicISub:1538 case SPIRV::OpAtomicUMin:1539 case SPIRV::OpAtomicUMax:1540 case SPIRV::OpAtomicSMin:1541 case SPIRV::OpAtomicSMax:1542 case SPIRV::OpAtomicAnd:1543 case SPIRV::OpAtomicOr:1544 case SPIRV::OpAtomicXor: {1545 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();1546 const MachineInstr *InstrPtr = &MI;1547 if (MI.getOpcode() == SPIRV::OpAtomicStore) {1548 assert(MI.getOperand(3).isReg());1549 InstrPtr = MRI.getVRegDef(MI.getOperand(3).getReg());1550 assert(InstrPtr && "Unexpected type instruction for OpAtomicStore");1551 }1552 assert(InstrPtr->getOperand(1).isReg() && "Unexpected operand in atomic");1553 Register TypeReg = InstrPtr->getOperand(1).getReg();1554 SPIRVType *TypeDef = MRI.getVRegDef(TypeReg);1555 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {1556 unsigned BitWidth = TypeDef->getOperand(1).getImm();1557 if (BitWidth == 64)1558 Reqs.addCapability(SPIRV::Capability::Int64Atomics);1559 }1560 break;1561 }1562 case SPIRV::OpGroupNonUniformIAdd:1563 case SPIRV::OpGroupNonUniformFAdd:1564 case SPIRV::OpGroupNonUniformIMul:1565 case SPIRV::OpGroupNonUniformFMul:1566 case SPIRV::OpGroupNonUniformSMin:1567 case SPIRV::OpGroupNonUniformUMin:1568 case SPIRV::OpGroupNonUniformFMin:1569 case SPIRV::OpGroupNonUniformSMax:1570 case SPIRV::OpGroupNonUniformUMax:1571 case SPIRV::OpGroupNonUniformFMax:1572 case SPIRV::OpGroupNonUniformBitwiseAnd:1573 case SPIRV::OpGroupNonUniformBitwiseOr:1574 case SPIRV::OpGroupNonUniformBitwiseXor:1575 case SPIRV::OpGroupNonUniformLogicalAnd:1576 case SPIRV::OpGroupNonUniformLogicalOr:1577 case SPIRV::OpGroupNonUniformLogicalXor: {1578 assert(MI.getOperand(3).isImm());1579 int64_t GroupOp = MI.getOperand(3).getImm();1580 switch (GroupOp) {1581 case SPIRV::GroupOperation::Reduce:1582 case SPIRV::GroupOperation::InclusiveScan:1583 case SPIRV::GroupOperation::ExclusiveScan:1584 Reqs.addCapability(SPIRV::Capability::GroupNonUniformArithmetic);1585 break;1586 case SPIRV::GroupOperation::ClusteredReduce:1587 Reqs.addCapability(SPIRV::Capability::GroupNonUniformClustered);1588 break;1589 case SPIRV::GroupOperation::PartitionedReduceNV:1590 case SPIRV::GroupOperation::PartitionedInclusiveScanNV:1591 case SPIRV::GroupOperation::PartitionedExclusiveScanNV:1592 Reqs.addCapability(SPIRV::Capability::GroupNonUniformPartitionedNV);1593 break;1594 }1595 break;1596 }1597 case SPIRV::OpGroupNonUniformShuffle:1598 case SPIRV::OpGroupNonUniformShuffleXor:1599 Reqs.addCapability(SPIRV::Capability::GroupNonUniformShuffle);1600 break;1601 case SPIRV::OpGroupNonUniformShuffleUp:1602 case SPIRV::OpGroupNonUniformShuffleDown:1603 Reqs.addCapability(SPIRV::Capability::GroupNonUniformShuffleRelative);1604 break;1605 case SPIRV::OpGroupAll:1606 case SPIRV::OpGroupAny:1607 case SPIRV::OpGroupBroadcast:1608 case SPIRV::OpGroupIAdd:1609 case SPIRV::OpGroupFAdd:1610 case SPIRV::OpGroupFMin:1611 case SPIRV::OpGroupUMin:1612 case SPIRV::OpGroupSMin:1613 case SPIRV::OpGroupFMax:1614 case SPIRV::OpGroupUMax:1615 case SPIRV::OpGroupSMax:1616 Reqs.addCapability(SPIRV::Capability::Groups);1617 break;1618 case SPIRV::OpGroupNonUniformElect:1619 Reqs.addCapability(SPIRV::Capability::GroupNonUniform);1620 break;1621 case SPIRV::OpGroupNonUniformAll:1622 case SPIRV::OpGroupNonUniformAny:1623 case SPIRV::OpGroupNonUniformAllEqual:1624 Reqs.addCapability(SPIRV::Capability::GroupNonUniformVote);1625 break;1626 case SPIRV::OpGroupNonUniformBroadcast:1627 case SPIRV::OpGroupNonUniformBroadcastFirst:1628 case SPIRV::OpGroupNonUniformBallot:1629 case SPIRV::OpGroupNonUniformInverseBallot:1630 case SPIRV::OpGroupNonUniformBallotBitExtract:1631 case SPIRV::OpGroupNonUniformBallotBitCount:1632 case SPIRV::OpGroupNonUniformBallotFindLSB:1633 case SPIRV::OpGroupNonUniformBallotFindMSB:1634 Reqs.addCapability(SPIRV::Capability::GroupNonUniformBallot);1635 break;1636 case SPIRV::OpSubgroupShuffleINTEL:1637 case SPIRV::OpSubgroupShuffleDownINTEL:1638 case SPIRV::OpSubgroupShuffleUpINTEL:1639 case SPIRV::OpSubgroupShuffleXorINTEL:1640 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {1641 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_subgroups);1642 Reqs.addCapability(SPIRV::Capability::SubgroupShuffleINTEL);1643 }1644 break;1645 case SPIRV::OpSubgroupBlockReadINTEL:1646 case SPIRV::OpSubgroupBlockWriteINTEL:1647 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {1648 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_subgroups);1649 Reqs.addCapability(SPIRV::Capability::SubgroupBufferBlockIOINTEL);1650 }1651 break;1652 case SPIRV::OpSubgroupImageBlockReadINTEL:1653 case SPIRV::OpSubgroupImageBlockWriteINTEL:1654 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {1655 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_subgroups);1656 Reqs.addCapability(SPIRV::Capability::SubgroupImageBlockIOINTEL);1657 }1658 break;1659 case SPIRV::OpSubgroupImageMediaBlockReadINTEL:1660 case SPIRV::OpSubgroupImageMediaBlockWriteINTEL:1661 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_media_block_io)) {1662 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_media_block_io);1663 Reqs.addCapability(SPIRV::Capability::SubgroupImageMediaBlockIOINTEL);1664 }1665 break;1666 case SPIRV::OpAssumeTrueKHR:1667 case SPIRV::OpExpectKHR:1668 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) {1669 Reqs.addExtension(SPIRV::Extension::SPV_KHR_expect_assume);1670 Reqs.addCapability(SPIRV::Capability::ExpectAssumeKHR);1671 }1672 break;1673 case SPIRV::OpPtrCastToCrossWorkgroupINTEL:1674 case SPIRV::OpCrossWorkgroupCastToPtrINTEL:1675 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes)) {1676 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes);1677 Reqs.addCapability(SPIRV::Capability::USMStorageClassesINTEL);1678 }1679 break;1680 case SPIRV::OpConstantFunctionPointerINTEL:1681 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)) {1682 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_function_pointers);1683 Reqs.addCapability(SPIRV::Capability::FunctionPointersINTEL);1684 }1685 break;1686 case SPIRV::OpGroupNonUniformRotateKHR:1687 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_subgroup_rotate))1688 report_fatal_error("OpGroupNonUniformRotateKHR instruction requires the "1689 "following SPIR-V extension: SPV_KHR_subgroup_rotate",1690 false);1691 Reqs.addExtension(SPIRV::Extension::SPV_KHR_subgroup_rotate);1692 Reqs.addCapability(SPIRV::Capability::GroupNonUniformRotateKHR);1693 Reqs.addCapability(SPIRV::Capability::GroupNonUniform);1694 break;1695 case SPIRV::OpFixedCosALTERA:1696 case SPIRV::OpFixedSinALTERA:1697 case SPIRV::OpFixedCosPiALTERA:1698 case SPIRV::OpFixedSinPiALTERA:1699 case SPIRV::OpFixedExpALTERA:1700 case SPIRV::OpFixedLogALTERA:1701 case SPIRV::OpFixedRecipALTERA:1702 case SPIRV::OpFixedSqrtALTERA:1703 case SPIRV::OpFixedSinCosALTERA:1704 case SPIRV::OpFixedSinCosPiALTERA:1705 case SPIRV::OpFixedRsqrtALTERA:1706 if (!ST.canUseExtension(1707 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point))1708 report_fatal_error("This instruction requires the "1709 "following SPIR-V extension: "1710 "SPV_ALTERA_arbitrary_precision_fixed_point",1711 false);1712 Reqs.addExtension(1713 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point);1714 Reqs.addCapability(SPIRV::Capability::ArbitraryPrecisionFixedPointALTERA);1715 break;1716 case SPIRV::OpGroupIMulKHR:1717 case SPIRV::OpGroupFMulKHR:1718 case SPIRV::OpGroupBitwiseAndKHR:1719 case SPIRV::OpGroupBitwiseOrKHR:1720 case SPIRV::OpGroupBitwiseXorKHR:1721 case SPIRV::OpGroupLogicalAndKHR:1722 case SPIRV::OpGroupLogicalOrKHR:1723 case SPIRV::OpGroupLogicalXorKHR:1724 if (ST.canUseExtension(1725 SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {1726 Reqs.addExtension(SPIRV::Extension::SPV_KHR_uniform_group_instructions);1727 Reqs.addCapability(SPIRV::Capability::GroupUniformArithmeticKHR);1728 }1729 break;1730 case SPIRV::OpReadClockKHR:1731 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_shader_clock))1732 report_fatal_error("OpReadClockKHR instruction requires the "1733 "following SPIR-V extension: SPV_KHR_shader_clock",1734 false);1735 Reqs.addExtension(SPIRV::Extension::SPV_KHR_shader_clock);1736 Reqs.addCapability(SPIRV::Capability::ShaderClockKHR);1737 break;1738 case SPIRV::OpFunctionPointerCallINTEL:1739 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)) {1740 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_function_pointers);1741 Reqs.addCapability(SPIRV::Capability::FunctionPointersINTEL);1742 }1743 break;1744 case SPIRV::OpAtomicFAddEXT:1745 case SPIRV::OpAtomicFMinEXT:1746 case SPIRV::OpAtomicFMaxEXT:1747 AddAtomicFloatRequirements(MI, Reqs, ST);1748 break;1749 case SPIRV::OpConvertBF16ToFINTEL:1750 case SPIRV::OpConvertFToBF16INTEL:1751 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_bfloat16_conversion)) {1752 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_bfloat16_conversion);1753 Reqs.addCapability(SPIRV::Capability::BFloat16ConversionINTEL);1754 }1755 break;1756 case SPIRV::OpRoundFToTF32INTEL:1757 if (ST.canUseExtension(1758 SPIRV::Extension::SPV_INTEL_tensor_float32_conversion)) {1759 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_tensor_float32_conversion);1760 Reqs.addCapability(SPIRV::Capability::TensorFloat32RoundingINTEL);1761 }1762 break;1763 case SPIRV::OpVariableLengthArrayINTEL:1764 case SPIRV::OpSaveMemoryINTEL:1765 case SPIRV::OpRestoreMemoryINTEL:1766 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) {1767 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_variable_length_array);1768 Reqs.addCapability(SPIRV::Capability::VariableLengthArrayINTEL);1769 }1770 break;1771 case SPIRV::OpAsmTargetINTEL:1772 case SPIRV::OpAsmINTEL:1773 case SPIRV::OpAsmCallINTEL:1774 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_inline_assembly)) {1775 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_inline_assembly);1776 Reqs.addCapability(SPIRV::Capability::AsmINTEL);1777 }1778 break;1779 case SPIRV::OpTypeCooperativeMatrixKHR: {1780 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))1781 report_fatal_error(1782 "OpTypeCooperativeMatrixKHR type requires the "1783 "following SPIR-V extension: SPV_KHR_cooperative_matrix",1784 false);1785 Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);1786 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);1787 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();1788 SPIRVType *TypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());1789 if (isBFloat16Type(TypeDef))1790 Reqs.addCapability(SPIRV::Capability::BFloat16CooperativeMatrixKHR);1791 break;1792 }1793 case SPIRV::OpArithmeticFenceEXT:1794 if (!ST.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence))1795 report_fatal_error("OpArithmeticFenceEXT requires the "1796 "following SPIR-V extension: SPV_EXT_arithmetic_fence",1797 false);1798 Reqs.addExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence);1799 Reqs.addCapability(SPIRV::Capability::ArithmeticFenceEXT);1800 break;1801 case SPIRV::OpControlBarrierArriveINTEL:1802 case SPIRV::OpControlBarrierWaitINTEL:1803 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_split_barrier)) {1804 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_split_barrier);1805 Reqs.addCapability(SPIRV::Capability::SplitBarrierINTEL);1806 }1807 break;1808 case SPIRV::OpCooperativeMatrixMulAddKHR: {1809 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))1810 report_fatal_error("Cooperative matrix instructions require the "1811 "following SPIR-V extension: "1812 "SPV_KHR_cooperative_matrix",1813 false);1814 Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);1815 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);1816 constexpr unsigned MulAddMaxSize = 6;1817 if (MI.getNumOperands() != MulAddMaxSize)1818 break;1819 const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm();1820 if (CoopOperands &1821 SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {1822 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))1823 report_fatal_error("MatrixAAndBTF32ComponentsINTEL type interpretation "1824 "require the following SPIR-V extension: "1825 "SPV_INTEL_joint_matrix",1826 false);1827 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);1828 Reqs.addCapability(1829 SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);1830 }1831 if (CoopOperands & SPIRV::CooperativeMatrixOperands::1832 MatrixAAndBBFloat16ComponentsINTEL ||1833 CoopOperands &1834 SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||1835 CoopOperands & SPIRV::CooperativeMatrixOperands::1836 MatrixResultBFloat16ComponentsINTEL) {1837 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))1838 report_fatal_error("***BF16ComponentsINTEL type interpretations "1839 "require the following SPIR-V extension: "1840 "SPV_INTEL_joint_matrix",1841 false);1842 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);1843 Reqs.addCapability(1844 SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);1845 }1846 break;1847 }1848 case SPIRV::OpCooperativeMatrixLoadKHR:1849 case SPIRV::OpCooperativeMatrixStoreKHR:1850 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:1851 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:1852 case SPIRV::OpCooperativeMatrixPrefetchINTEL: {1853 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))1854 report_fatal_error("Cooperative matrix instructions require the "1855 "following SPIR-V extension: "1856 "SPV_KHR_cooperative_matrix",1857 false);1858 Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);1859 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);1860 1861 // Check Layout operand in case if it's not a standard one and add the1862 // appropriate capability.1863 std::unordered_map<unsigned, unsigned> LayoutToInstMap = {1864 {SPIRV::OpCooperativeMatrixLoadKHR, 3},1865 {SPIRV::OpCooperativeMatrixStoreKHR, 2},1866 {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5},1867 {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4},1868 {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4}};1869 1870 const auto OpCode = MI.getOpcode();1871 const unsigned LayoutNum = LayoutToInstMap[OpCode];1872 Register RegLayout = MI.getOperand(LayoutNum).getReg();1873 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();1874 MachineInstr *MILayout = MRI.getUniqueVRegDef(RegLayout);1875 if (MILayout->getOpcode() == SPIRV::OpConstantI) {1876 const unsigned LayoutVal = MILayout->getOperand(2).getImm();1877 if (LayoutVal ==1878 static_cast<unsigned>(SPIRV::CooperativeMatrixLayout::PackedINTEL)) {1879 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))1880 report_fatal_error("PackedINTEL layout require the following SPIR-V "1881 "extension: SPV_INTEL_joint_matrix",1882 false);1883 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);1884 Reqs.addCapability(SPIRV::Capability::PackedCooperativeMatrixINTEL);1885 }1886 }1887 1888 // Nothing to do.1889 if (OpCode == SPIRV::OpCooperativeMatrixLoadKHR ||1890 OpCode == SPIRV::OpCooperativeMatrixStoreKHR)1891 break;1892 1893 std::string InstName;1894 switch (OpCode) {1895 case SPIRV::OpCooperativeMatrixPrefetchINTEL:1896 InstName = "OpCooperativeMatrixPrefetchINTEL";1897 break;1898 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:1899 InstName = "OpCooperativeMatrixLoadCheckedINTEL";1900 break;1901 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:1902 InstName = "OpCooperativeMatrixStoreCheckedINTEL";1903 break;1904 }1905 1906 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) {1907 const std::string ErrorMsg =1908 InstName + " instruction requires the "1909 "following SPIR-V extension: SPV_INTEL_joint_matrix";1910 report_fatal_error(ErrorMsg.c_str(), false);1911 }1912 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);1913 if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {1914 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL);1915 break;1916 }1917 Reqs.addCapability(1918 SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);1919 break;1920 }1921 case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:1922 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))1923 report_fatal_error("OpCooperativeMatrixConstructCheckedINTEL "1924 "instructions require the following SPIR-V extension: "1925 "SPV_INTEL_joint_matrix",1926 false);1927 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);1928 Reqs.addCapability(1929 SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);1930 break;1931 case SPIRV::OpReadPipeBlockingALTERA:1932 case SPIRV::OpWritePipeBlockingALTERA:1933 if (ST.canUseExtension(SPIRV::Extension::SPV_ALTERA_blocking_pipes)) {1934 Reqs.addExtension(SPIRV::Extension::SPV_ALTERA_blocking_pipes);1935 Reqs.addCapability(SPIRV::Capability::BlockingPipesALTERA);1936 }1937 break;1938 case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:1939 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))1940 report_fatal_error("OpCooperativeMatrixGetElementCoordINTEL requires the "1941 "following SPIR-V extension: SPV_INTEL_joint_matrix",1942 false);1943 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);1944 Reqs.addCapability(1945 SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);1946 break;1947 case SPIRV::OpConvertHandleToImageINTEL:1948 case SPIRV::OpConvertHandleToSamplerINTEL:1949 case SPIRV::OpConvertHandleToSampledImageINTEL: {1950 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_bindless_images))1951 report_fatal_error("OpConvertHandleTo[Image/Sampler/SampledImage]INTEL "1952 "instructions require the following SPIR-V extension: "1953 "SPV_INTEL_bindless_images",1954 false);1955 SPIRVGlobalRegistry *GR = ST.getSPIRVGlobalRegistry();1956 SPIRV::AddressingModel::AddressingModel AddrModel = MAI.Addr;1957 SPIRVType *TyDef = GR->getSPIRVTypeForVReg(MI.getOperand(1).getReg());1958 if (MI.getOpcode() == SPIRV::OpConvertHandleToImageINTEL &&1959 TyDef->getOpcode() != SPIRV::OpTypeImage) {1960 report_fatal_error("Incorrect return type for the instruction "1961 "OpConvertHandleToImageINTEL",1962 false);1963 } else if (MI.getOpcode() == SPIRV::OpConvertHandleToSamplerINTEL &&1964 TyDef->getOpcode() != SPIRV::OpTypeSampler) {1965 report_fatal_error("Incorrect return type for the instruction "1966 "OpConvertHandleToSamplerINTEL",1967 false);1968 } else if (MI.getOpcode() == SPIRV::OpConvertHandleToSampledImageINTEL &&1969 TyDef->getOpcode() != SPIRV::OpTypeSampledImage) {1970 report_fatal_error("Incorrect return type for the instruction "1971 "OpConvertHandleToSampledImageINTEL",1972 false);1973 }1974 SPIRVType *SpvTy = GR->getSPIRVTypeForVReg(MI.getOperand(2).getReg());1975 unsigned Bitwidth = GR->getScalarOrVectorBitWidth(SpvTy);1976 if (!(Bitwidth == 32 && AddrModel == SPIRV::AddressingModel::Physical32) &&1977 !(Bitwidth == 64 && AddrModel == SPIRV::AddressingModel::Physical64)) {1978 report_fatal_error(1979 "Parameter value must be a 32-bit scalar in case of "1980 "Physical32 addressing model or a 64-bit scalar in case of "1981 "Physical64 addressing model",1982 false);1983 }1984 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_bindless_images);1985 Reqs.addCapability(SPIRV::Capability::BindlessImagesINTEL);1986 break;1987 }1988 case SPIRV::OpSubgroup2DBlockLoadINTEL:1989 case SPIRV::OpSubgroup2DBlockLoadTransposeINTEL:1990 case SPIRV::OpSubgroup2DBlockLoadTransformINTEL:1991 case SPIRV::OpSubgroup2DBlockPrefetchINTEL:1992 case SPIRV::OpSubgroup2DBlockStoreINTEL: {1993 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_2d_block_io))1994 report_fatal_error("OpSubgroup2DBlock[Load/LoadTranspose/LoadTransform/"1995 "Prefetch/Store]INTEL instructions require the "1996 "following SPIR-V extension: SPV_INTEL_2d_block_io",1997 false);1998 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_2d_block_io);1999 Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockIOINTEL);2000 2001 const auto OpCode = MI.getOpcode();2002 if (OpCode == SPIRV::OpSubgroup2DBlockLoadTransposeINTEL) {2003 Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockTransposeINTEL);2004 break;2005 }2006 if (OpCode == SPIRV::OpSubgroup2DBlockLoadTransformINTEL) {2007 Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockTransformINTEL);2008 break;2009 }2010 break;2011 }2012 case SPIRV::OpKill: {2013 Reqs.addCapability(SPIRV::Capability::Shader);2014 } break;2015 case SPIRV::OpDemoteToHelperInvocation:2016 Reqs.addCapability(SPIRV::Capability::DemoteToHelperInvocation);2017 2018 if (ST.canUseExtension(2019 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation)) {2020 if (!ST.isAtLeastSPIRVVer(llvm::VersionTuple(1, 6)))2021 Reqs.addExtension(2022 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation);2023 }2024 break;2025 case SPIRV::OpSDot:2026 case SPIRV::OpUDot:2027 case SPIRV::OpSUDot:2028 case SPIRV::OpSDotAccSat:2029 case SPIRV::OpUDotAccSat:2030 case SPIRV::OpSUDotAccSat:2031 AddDotProductRequirements(MI, Reqs, ST);2032 break;2033 case SPIRV::OpImageRead: {2034 Register ImageReg = MI.getOperand(2).getReg();2035 SPIRVType *TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(2036 ImageReg, const_cast<MachineFunction *>(MI.getMF()));2037 // OpImageRead and OpImageWrite can use Unknown Image Formats2038 // when the Kernel capability is declared. In the OpenCL environment we are2039 // not allowed to produce2040 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see2041 // https://github.com/KhronosGroup/SPIRV-Headers/issues/4872042 2043 if (isImageTypeWithUnknownFormat(TypeDef) && ST.isShader())2044 Reqs.addCapability(SPIRV::Capability::StorageImageReadWithoutFormat);2045 break;2046 }2047 case SPIRV::OpImageWrite: {2048 Register ImageReg = MI.getOperand(0).getReg();2049 SPIRVType *TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(2050 ImageReg, const_cast<MachineFunction *>(MI.getMF()));2051 // OpImageRead and OpImageWrite can use Unknown Image Formats2052 // when the Kernel capability is declared. In the OpenCL environment we are2053 // not allowed to produce2054 // StorageImageReadWithoutFormat/StorageImageWriteWithoutFormat, see2055 // https://github.com/KhronosGroup/SPIRV-Headers/issues/4872056 2057 if (isImageTypeWithUnknownFormat(TypeDef) && ST.isShader())2058 Reqs.addCapability(SPIRV::Capability::StorageImageWriteWithoutFormat);2059 break;2060 }2061 case SPIRV::OpTypeStructContinuedINTEL:2062 case SPIRV::OpConstantCompositeContinuedINTEL:2063 case SPIRV::OpSpecConstantCompositeContinuedINTEL:2064 case SPIRV::OpCompositeConstructContinuedINTEL: {2065 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_long_composites))2066 report_fatal_error(2067 "Continued instructions require the "2068 "following SPIR-V extension: SPV_INTEL_long_composites",2069 false);2070 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_long_composites);2071 Reqs.addCapability(SPIRV::Capability::LongCompositesINTEL);2072 break;2073 }2074 case SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL: {2075 if (!ST.canUseExtension(2076 SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate))2077 report_fatal_error(2078 "OpSubgroupMatrixMultiplyAccumulateINTEL instruction requires the "2079 "following SPIR-V "2080 "extension: SPV_INTEL_subgroup_matrix_multiply_accumulate",2081 false);2082 Reqs.addExtension(2083 SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate);2084 Reqs.addCapability(2085 SPIRV::Capability::SubgroupMatrixMultiplyAccumulateINTEL);2086 break;2087 }2088 case SPIRV::OpBitwiseFunctionINTEL: {2089 if (!ST.canUseExtension(2090 SPIRV::Extension::SPV_INTEL_ternary_bitwise_function))2091 report_fatal_error(2092 "OpBitwiseFunctionINTEL instruction requires the following SPIR-V "2093 "extension: SPV_INTEL_ternary_bitwise_function",2094 false);2095 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_ternary_bitwise_function);2096 Reqs.addCapability(SPIRV::Capability::TernaryBitwiseFunctionINTEL);2097 break;2098 }2099 case SPIRV::OpCopyMemorySized: {2100 Reqs.addCapability(SPIRV::Capability::Addresses);2101 // TODO: Add UntypedPointersKHR when implemented.2102 break;2103 }2104 case SPIRV::OpPredicatedLoadINTEL:2105 case SPIRV::OpPredicatedStoreINTEL: {2106 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_predicated_io))2107 report_fatal_error(2108 "OpPredicated[Load/Store]INTEL instructions require "2109 "the following SPIR-V extension: SPV_INTEL_predicated_io",2110 false);2111 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_predicated_io);2112 Reqs.addCapability(SPIRV::Capability::PredicatedIOINTEL);2113 break;2114 }2115 case SPIRV::OpFAddS:2116 case SPIRV::OpFSubS:2117 case SPIRV::OpFMulS:2118 case SPIRV::OpFDivS:2119 case SPIRV::OpFRemS:2120 case SPIRV::OpFMod:2121 case SPIRV::OpFNegate:2122 case SPIRV::OpFAddV:2123 case SPIRV::OpFSubV:2124 case SPIRV::OpFMulV:2125 case SPIRV::OpFDivV:2126 case SPIRV::OpFRemV:2127 case SPIRV::OpFNegateV: {2128 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();2129 SPIRVType *TypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());2130 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)2131 TypeDef = MRI.getVRegDef(TypeDef->getOperand(1).getReg());2132 if (isBFloat16Type(TypeDef)) {2133 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))2134 report_fatal_error(2135 "Arithmetic instructions with bfloat16 arguments require the "2136 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",2137 false);2138 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);2139 Reqs.addCapability(SPIRV::Capability::BFloat16ArithmeticINTEL);2140 }2141 break;2142 }2143 case SPIRV::OpOrdered:2144 case SPIRV::OpUnordered:2145 case SPIRV::OpFOrdEqual:2146 case SPIRV::OpFOrdNotEqual:2147 case SPIRV::OpFOrdLessThan:2148 case SPIRV::OpFOrdLessThanEqual:2149 case SPIRV::OpFOrdGreaterThan:2150 case SPIRV::OpFOrdGreaterThanEqual:2151 case SPIRV::OpFUnordEqual:2152 case SPIRV::OpFUnordNotEqual:2153 case SPIRV::OpFUnordLessThan:2154 case SPIRV::OpFUnordLessThanEqual:2155 case SPIRV::OpFUnordGreaterThan:2156 case SPIRV::OpFUnordGreaterThanEqual: {2157 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();2158 MachineInstr *OperandDef = MRI.getVRegDef(MI.getOperand(2).getReg());2159 SPIRVType *TypeDef = MRI.getVRegDef(OperandDef->getOperand(1).getReg());2160 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)2161 TypeDef = MRI.getVRegDef(TypeDef->getOperand(1).getReg());2162 if (isBFloat16Type(TypeDef)) {2163 if (!ST.canUseExtension(SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))2164 report_fatal_error(2165 "Relational instructions with bfloat16 arguments require the "2166 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",2167 false);2168 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);2169 Reqs.addCapability(SPIRV::Capability::BFloat16ArithmeticINTEL);2170 }2171 break;2172 }2173 case SPIRV::OpDPdxCoarse:2174 case SPIRV::OpDPdyCoarse: {2175 Reqs.addCapability(SPIRV::Capability::DerivativeControl);2176 break;2177 }2178 2179 default:2180 break;2181 }2182 2183 // If we require capability Shader, then we can remove the requirement for2184 // the BitInstructions capability, since Shader is a superset capability2185 // of BitInstructions.2186 Reqs.removeCapabilityIf(SPIRV::Capability::BitInstructions,2187 SPIRV::Capability::Shader);2188}2189 2190static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI,2191 MachineModuleInfo *MMI, const SPIRVSubtarget &ST) {2192 // Collect requirements for existing instructions.2193 for (const Function &F : M) {2194 MachineFunction *MF = MMI->getMachineFunction(F);2195 if (!MF)2196 continue;2197 for (const MachineBasicBlock &MBB : *MF)2198 for (const MachineInstr &MI : MBB)2199 addInstrRequirements(MI, MAI, ST);2200 }2201 // Collect requirements for OpExecutionMode instructions.2202 auto Node = M.getNamedMetadata("spirv.ExecutionMode");2203 if (Node) {2204 bool RequireFloatControls = false, RequireIntelFloatControls2 = false,2205 RequireKHRFloatControls2 = false,2206 VerLower14 = !ST.isAtLeastSPIRVVer(VersionTuple(1, 4));2207 bool HasIntelFloatControls2 =2208 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_float_controls2);2209 bool HasKHRFloatControls2 =2210 ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);2211 for (unsigned i = 0; i < Node->getNumOperands(); i++) {2212 MDNode *MDN = cast<MDNode>(Node->getOperand(i));2213 const MDOperand &MDOp = MDN->getOperand(1);2214 if (auto *CMeta = dyn_cast<ConstantAsMetadata>(MDOp)) {2215 Constant *C = CMeta->getValue();2216 if (ConstantInt *Const = dyn_cast<ConstantInt>(C)) {2217 auto EM = Const->getZExtValue();2218 // SPV_KHR_float_controls is not available until v1.4:2219 // add SPV_KHR_float_controls if the version is too low2220 switch (EM) {2221 case SPIRV::ExecutionMode::DenormPreserve:2222 case SPIRV::ExecutionMode::DenormFlushToZero:2223 case SPIRV::ExecutionMode::RoundingModeRTE:2224 case SPIRV::ExecutionMode::RoundingModeRTZ:2225 RequireFloatControls = VerLower14;2226 MAI.Reqs.getAndAddRequirements(2227 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);2228 break;2229 case SPIRV::ExecutionMode::RoundingModeRTPINTEL:2230 case SPIRV::ExecutionMode::RoundingModeRTNINTEL:2231 case SPIRV::ExecutionMode::FloatingPointModeALTINTEL:2232 case SPIRV::ExecutionMode::FloatingPointModeIEEEINTEL:2233 if (HasIntelFloatControls2) {2234 RequireIntelFloatControls2 = true;2235 MAI.Reqs.getAndAddRequirements(2236 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);2237 }2238 break;2239 case SPIRV::ExecutionMode::FPFastMathDefault: {2240 if (HasKHRFloatControls2) {2241 RequireKHRFloatControls2 = true;2242 MAI.Reqs.getAndAddRequirements(2243 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);2244 }2245 break;2246 }2247 case SPIRV::ExecutionMode::ContractionOff:2248 case SPIRV::ExecutionMode::SignedZeroInfNanPreserve:2249 if (HasKHRFloatControls2) {2250 RequireKHRFloatControls2 = true;2251 MAI.Reqs.getAndAddRequirements(2252 SPIRV::OperandCategory::ExecutionModeOperand,2253 SPIRV::ExecutionMode::FPFastMathDefault, ST);2254 } else {2255 MAI.Reqs.getAndAddRequirements(2256 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);2257 }2258 break;2259 default:2260 MAI.Reqs.getAndAddRequirements(2261 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);2262 }2263 }2264 }2265 }2266 if (RequireFloatControls &&2267 ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls))2268 MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_float_controls);2269 if (RequireIntelFloatControls2)2270 MAI.Reqs.addExtension(SPIRV::Extension::SPV_INTEL_float_controls2);2271 if (RequireKHRFloatControls2)2272 MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_float_controls2);2273 }2274 for (const Function &F : M) {2275 if (F.isDeclaration())2276 continue;2277 if (F.getMetadata("reqd_work_group_size"))2278 MAI.Reqs.getAndAddRequirements(2279 SPIRV::OperandCategory::ExecutionModeOperand,2280 SPIRV::ExecutionMode::LocalSize, ST);2281 if (F.getFnAttribute("hlsl.numthreads").isValid()) {2282 MAI.Reqs.getAndAddRequirements(2283 SPIRV::OperandCategory::ExecutionModeOperand,2284 SPIRV::ExecutionMode::LocalSize, ST);2285 }2286 if (F.getFnAttribute("enable-maximal-reconvergence").getValueAsBool()) {2287 MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_maximal_reconvergence);2288 }2289 if (F.getMetadata("work_group_size_hint"))2290 MAI.Reqs.getAndAddRequirements(2291 SPIRV::OperandCategory::ExecutionModeOperand,2292 SPIRV::ExecutionMode::LocalSizeHint, ST);2293 if (F.getMetadata("intel_reqd_sub_group_size"))2294 MAI.Reqs.getAndAddRequirements(2295 SPIRV::OperandCategory::ExecutionModeOperand,2296 SPIRV::ExecutionMode::SubgroupSize, ST);2297 if (F.getMetadata("max_work_group_size"))2298 MAI.Reqs.getAndAddRequirements(2299 SPIRV::OperandCategory::ExecutionModeOperand,2300 SPIRV::ExecutionMode::MaxWorkgroupSizeINTEL, ST);2301 if (F.getMetadata("vec_type_hint"))2302 MAI.Reqs.getAndAddRequirements(2303 SPIRV::OperandCategory::ExecutionModeOperand,2304 SPIRV::ExecutionMode::VecTypeHint, ST);2305 2306 if (F.hasOptNone()) {2307 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_optnone)) {2308 MAI.Reqs.addExtension(SPIRV::Extension::SPV_INTEL_optnone);2309 MAI.Reqs.addCapability(SPIRV::Capability::OptNoneINTEL);2310 } else if (ST.canUseExtension(SPIRV::Extension::SPV_EXT_optnone)) {2311 MAI.Reqs.addExtension(SPIRV::Extension::SPV_EXT_optnone);2312 MAI.Reqs.addCapability(SPIRV::Capability::OptNoneEXT);2313 }2314 }2315 }2316}2317 2318static unsigned getFastMathFlags(const MachineInstr &I,2319 const SPIRVSubtarget &ST) {2320 unsigned Flags = SPIRV::FPFastMathMode::None;2321 bool CanUseKHRFloatControls2 =2322 ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);2323 if (I.getFlag(MachineInstr::MIFlag::FmNoNans))2324 Flags |= SPIRV::FPFastMathMode::NotNaN;2325 if (I.getFlag(MachineInstr::MIFlag::FmNoInfs))2326 Flags |= SPIRV::FPFastMathMode::NotInf;2327 if (I.getFlag(MachineInstr::MIFlag::FmNsz))2328 Flags |= SPIRV::FPFastMathMode::NSZ;2329 if (I.getFlag(MachineInstr::MIFlag::FmArcp))2330 Flags |= SPIRV::FPFastMathMode::AllowRecip;2331 if (I.getFlag(MachineInstr::MIFlag::FmContract) && CanUseKHRFloatControls2)2332 Flags |= SPIRV::FPFastMathMode::AllowContract;2333 if (I.getFlag(MachineInstr::MIFlag::FmReassoc)) {2334 if (CanUseKHRFloatControls2)2335 // LLVM reassoc maps to SPIRV transform, see2336 // https://github.com/KhronosGroup/SPIRV-Registry/issues/326 for details.2337 // Because we are enabling AllowTransform, we must enable AllowReassoc and2338 // AllowContract too, as required by SPIRV spec. Also, we used to map2339 // MIFlag::FmReassoc to FPFastMathMode::Fast, which now should instead by2340 // replaced by turning all the other bits instead. Therefore, we're2341 // enabling every bit here except None and Fast.2342 Flags |= SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |2343 SPIRV::FPFastMathMode::NSZ | SPIRV::FPFastMathMode::AllowRecip |2344 SPIRV::FPFastMathMode::AllowTransform |2345 SPIRV::FPFastMathMode::AllowReassoc |2346 SPIRV::FPFastMathMode::AllowContract;2347 else2348 Flags |= SPIRV::FPFastMathMode::Fast;2349 }2350 2351 if (CanUseKHRFloatControls2) {2352 // Error out if SPIRV::FPFastMathMode::Fast is enabled.2353 assert(!(Flags & SPIRV::FPFastMathMode::Fast) &&2354 "SPIRV::FPFastMathMode::Fast is deprecated and should not be used "2355 "anymore.");2356 2357 // Error out if AllowTransform is enabled without AllowReassoc and2358 // AllowContract.2359 assert((!(Flags & SPIRV::FPFastMathMode::AllowTransform) ||2360 ((Flags & SPIRV::FPFastMathMode::AllowReassoc &&2361 Flags & SPIRV::FPFastMathMode::AllowContract))) &&2362 "SPIRV::FPFastMathMode::AllowTransform requires AllowReassoc and "2363 "AllowContract flags to be enabled as well.");2364 }2365 2366 return Flags;2367}2368 2369static bool isFastMathModeAvailable(const SPIRVSubtarget &ST) {2370 if (ST.isKernel())2371 return true;2372 if (ST.getSPIRVVersion() < VersionTuple(1, 2))2373 return false;2374 return ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);2375}2376 2377static void handleMIFlagDecoration(2378 MachineInstr &I, const SPIRVSubtarget &ST, const SPIRVInstrInfo &TII,2379 SPIRV::RequirementHandler &Reqs, const SPIRVGlobalRegistry *GR,2380 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec) {2381 if (I.getFlag(MachineInstr::MIFlag::NoSWrap) && TII.canUseNSW(I) &&2382 getSymbolicOperandRequirements(SPIRV::OperandCategory::DecorationOperand,2383 SPIRV::Decoration::NoSignedWrap, ST, Reqs)2384 .IsSatisfiable) {2385 buildOpDecorate(I.getOperand(0).getReg(), I, TII,2386 SPIRV::Decoration::NoSignedWrap, {});2387 }2388 if (I.getFlag(MachineInstr::MIFlag::NoUWrap) && TII.canUseNUW(I) &&2389 getSymbolicOperandRequirements(SPIRV::OperandCategory::DecorationOperand,2390 SPIRV::Decoration::NoUnsignedWrap, ST,2391 Reqs)2392 .IsSatisfiable) {2393 buildOpDecorate(I.getOperand(0).getReg(), I, TII,2394 SPIRV::Decoration::NoUnsignedWrap, {});2395 }2396 if (!TII.canUseFastMathFlags(2397 I, ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2)))2398 return;2399 2400 unsigned FMFlags = getFastMathFlags(I, ST);2401 if (FMFlags == SPIRV::FPFastMathMode::None) {2402 // We also need to check if any FPFastMathDefault info was set for the2403 // types used in this instruction.2404 if (FPFastMathDefaultInfoVec.empty())2405 return;2406 2407 // There are three types of instructions that can use fast math flags:2408 // 1. Arithmetic instructions (FAdd, FMul, FSub, FDiv, FRem, etc.)2409 // 2. Relational instructions (FCmp, FOrd, FUnord, etc.)2410 // 3. Extended instructions (ExtInst)2411 // For arithmetic instructions, the floating point type can be in the2412 // result type or in the operands, but they all must be the same.2413 // For the relational and logical instructions, the floating point type2414 // can only be in the operands 1 and 2, not the result type. Also, the2415 // operands must have the same type. For the extended instructions, the2416 // floating point type can be in the result type or in the operands. It's2417 // unclear if the operands and the result type must be the same. Let's2418 // assume they must be. Therefore, for 1. and 2., we can check the first2419 // operand type, and for 3. we can check the result type.2420 assert(I.getNumOperands() >= 3 && "Expected at least 3 operands");2421 Register ResReg = I.getOpcode() == SPIRV::OpExtInst2422 ? I.getOperand(1).getReg()2423 : I.getOperand(2).getReg();2424 SPIRVType *ResType = GR->getSPIRVTypeForVReg(ResReg, I.getMF());2425 const Type *Ty = GR->getTypeForSPIRVType(ResType);2426 Ty = Ty->isVectorTy() ? cast<VectorType>(Ty)->getElementType() : Ty;2427 2428 // Match instruction type with the FPFastMathDefaultInfoVec.2429 bool Emit = false;2430 for (SPIRV::FPFastMathDefaultInfo &Elem : FPFastMathDefaultInfoVec) {2431 if (Ty == Elem.Ty) {2432 FMFlags = Elem.FastMathFlags;2433 Emit = Elem.ContractionOff || Elem.SignedZeroInfNanPreserve ||2434 Elem.FPFastMathDefault;2435 break;2436 }2437 }2438 2439 if (FMFlags == SPIRV::FPFastMathMode::None && !Emit)2440 return;2441 }2442 if (isFastMathModeAvailable(ST)) {2443 Register DstReg = I.getOperand(0).getReg();2444 buildOpDecorate(DstReg, I, TII, SPIRV::Decoration::FPFastMathMode,2445 {FMFlags});2446 }2447}2448 2449// Walk all functions and add decorations related to MI flags.2450static void addDecorations(const Module &M, const SPIRVInstrInfo &TII,2451 MachineModuleInfo *MMI, const SPIRVSubtarget &ST,2452 SPIRV::ModuleAnalysisInfo &MAI,2453 const SPIRVGlobalRegistry *GR) {2454 for (const Function &F : M) {2455 MachineFunction *MF = MMI->getMachineFunction(F);2456 if (!MF)2457 continue;2458 2459 for (auto &MBB : *MF)2460 for (auto &MI : MBB)2461 handleMIFlagDecoration(MI, ST, TII, MAI.Reqs, GR,2462 MAI.FPFastMathDefaultInfoMap[&F]);2463 }2464}2465 2466static void addMBBNames(const Module &M, const SPIRVInstrInfo &TII,2467 MachineModuleInfo *MMI, const SPIRVSubtarget &ST,2468 SPIRV::ModuleAnalysisInfo &MAI) {2469 for (const Function &F : M) {2470 MachineFunction *MF = MMI->getMachineFunction(F);2471 if (!MF)2472 continue;2473 MachineRegisterInfo &MRI = MF->getRegInfo();2474 for (auto &MBB : *MF) {2475 if (!MBB.hasName() || MBB.empty())2476 continue;2477 // Emit basic block names.2478 Register Reg = MRI.createGenericVirtualRegister(LLT::scalar(64));2479 MRI.setRegClass(Reg, &SPIRV::IDRegClass);2480 buildOpName(Reg, MBB.getName(), *std::prev(MBB.end()), TII);2481 MCRegister GlobalReg = MAI.getOrCreateMBBRegister(MBB);2482 MAI.setRegisterAlias(MF, Reg, GlobalReg);2483 }2484 }2485}2486 2487// patching Instruction::PHI to SPIRV::OpPhi2488static void patchPhis(const Module &M, SPIRVGlobalRegistry *GR,2489 const SPIRVInstrInfo &TII, MachineModuleInfo *MMI) {2490 for (const Function &F : M) {2491 MachineFunction *MF = MMI->getMachineFunction(F);2492 if (!MF)2493 continue;2494 for (auto &MBB : *MF) {2495 for (MachineInstr &MI : MBB.phis()) {2496 MI.setDesc(TII.get(SPIRV::OpPhi));2497 Register ResTypeReg = GR->getSPIRVTypeID(2498 GR->getSPIRVTypeForVReg(MI.getOperand(0).getReg(), MF));2499 MI.insert(MI.operands_begin() + 1,2500 {MachineOperand::CreateReg(ResTypeReg, false)});2501 }2502 }2503 2504 MF->getProperties().setNoPHIs();2505 }2506}2507 2508static SPIRV::FPFastMathDefaultInfoVector &getOrCreateFPFastMathDefaultInfoVec(2509 const Module &M, SPIRV::ModuleAnalysisInfo &MAI, const Function *F) {2510 auto it = MAI.FPFastMathDefaultInfoMap.find(F);2511 if (it != MAI.FPFastMathDefaultInfoMap.end())2512 return it->second;2513 2514 // If the map does not contain the entry, create a new one. Initialize it to2515 // contain all 3 elements sorted by bit width of target type: {half, float,2516 // double}.2517 SPIRV::FPFastMathDefaultInfoVector FPFastMathDefaultInfoVec;2518 FPFastMathDefaultInfoVec.emplace_back(Type::getHalfTy(M.getContext()),2519 SPIRV::FPFastMathMode::None);2520 FPFastMathDefaultInfoVec.emplace_back(Type::getFloatTy(M.getContext()),2521 SPIRV::FPFastMathMode::None);2522 FPFastMathDefaultInfoVec.emplace_back(Type::getDoubleTy(M.getContext()),2523 SPIRV::FPFastMathMode::None);2524 return MAI.FPFastMathDefaultInfoMap[F] = std::move(FPFastMathDefaultInfoVec);2525}2526 2527static SPIRV::FPFastMathDefaultInfo &getFPFastMathDefaultInfo(2528 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec,2529 const Type *Ty) {2530 size_t BitWidth = Ty->getScalarSizeInBits();2531 int Index =2532 SPIRV::FPFastMathDefaultInfoVector::computeFPFastMathDefaultInfoVecIndex(2533 BitWidth);2534 assert(Index >= 0 && Index < 3 &&2535 "Expected FPFastMathDefaultInfo for half, float, or double");2536 assert(FPFastMathDefaultInfoVec.size() == 3 &&2537 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");2538 return FPFastMathDefaultInfoVec[Index];2539}2540 2541static void collectFPFastMathDefaults(const Module &M,2542 SPIRV::ModuleAnalysisInfo &MAI,2543 const SPIRVSubtarget &ST) {2544 if (!ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2))2545 return;2546 2547 // Store the FPFastMathDefaultInfo in the FPFastMathDefaultInfoMap.2548 // We need the entry point (function) as the key, and the target2549 // type and flags as the value.2550 // We also need to check ContractionOff and SignedZeroInfNanPreserve2551 // execution modes, as they are now deprecated and must be replaced2552 // with FPFastMathDefaultInfo.2553 auto Node = M.getNamedMetadata("spirv.ExecutionMode");2554 if (!Node)2555 return;2556 2557 for (unsigned i = 0; i < Node->getNumOperands(); i++) {2558 MDNode *MDN = cast<MDNode>(Node->getOperand(i));2559 assert(MDN->getNumOperands() >= 2 && "Expected at least 2 operands");2560 const Function *F = cast<Function>(2561 cast<ConstantAsMetadata>(MDN->getOperand(0))->getValue());2562 const auto EM =2563 cast<ConstantInt>(2564 cast<ConstantAsMetadata>(MDN->getOperand(1))->getValue())2565 ->getZExtValue();2566 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {2567 assert(MDN->getNumOperands() == 4 &&2568 "Expected 4 operands for FPFastMathDefault");2569 2570 const Type *T = cast<ValueAsMetadata>(MDN->getOperand(2))->getType();2571 unsigned Flags =2572 cast<ConstantInt>(2573 cast<ConstantAsMetadata>(MDN->getOperand(3))->getValue())2574 ->getZExtValue();2575 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =2576 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);2577 SPIRV::FPFastMathDefaultInfo &Info =2578 getFPFastMathDefaultInfo(FPFastMathDefaultInfoVec, T);2579 Info.FastMathFlags = Flags;2580 Info.FPFastMathDefault = true;2581 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {2582 assert(MDN->getNumOperands() == 2 &&2583 "Expected no operands for ContractionOff");2584 2585 // We need to save this info for every possible FP type, i.e. {half,2586 // float, double, fp128}.2587 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =2588 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);2589 for (SPIRV::FPFastMathDefaultInfo &Info : FPFastMathDefaultInfoVec) {2590 Info.ContractionOff = true;2591 }2592 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {2593 assert(MDN->getNumOperands() == 3 &&2594 "Expected 1 operand for SignedZeroInfNanPreserve");2595 unsigned TargetWidth =2596 cast<ConstantInt>(2597 cast<ConstantAsMetadata>(MDN->getOperand(2))->getValue())2598 ->getZExtValue();2599 // We need to save this info only for the FP type with TargetWidth.2600 SPIRV::FPFastMathDefaultInfoVector &FPFastMathDefaultInfoVec =2601 getOrCreateFPFastMathDefaultInfoVec(M, MAI, F);2602 int Index = SPIRV::FPFastMathDefaultInfoVector::2603 computeFPFastMathDefaultInfoVecIndex(TargetWidth);2604 assert(Index >= 0 && Index < 3 &&2605 "Expected FPFastMathDefaultInfo for half, float, or double");2606 assert(FPFastMathDefaultInfoVec.size() == 3 &&2607 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");2608 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;2609 }2610 }2611}2612 2613struct SPIRV::ModuleAnalysisInfo SPIRVModuleAnalysis::MAI;2614 2615void SPIRVModuleAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {2616 AU.addRequired<TargetPassConfig>();2617 AU.addRequired<MachineModuleInfoWrapperPass>();2618}2619 2620bool SPIRVModuleAnalysis::runOnModule(Module &M) {2621 SPIRVTargetMachine &TM =2622 getAnalysis<TargetPassConfig>().getTM<SPIRVTargetMachine>();2623 ST = TM.getSubtargetImpl();2624 GR = ST->getSPIRVGlobalRegistry();2625 TII = ST->getInstrInfo();2626 2627 MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();2628 2629 setBaseInfo(M);2630 2631 patchPhis(M, GR, *TII, MMI);2632 2633 addMBBNames(M, *TII, MMI, *ST, MAI);2634 collectFPFastMathDefaults(M, MAI, *ST);2635 addDecorations(M, *TII, MMI, *ST, MAI, GR);2636 2637 collectReqs(M, MAI, MMI, *ST);2638 2639 // Process type/const/global var/func decl instructions, number their2640 // destination registers from 0 to N, collect Extensions and Capabilities.2641 collectReqs(M, MAI, MMI, *ST);2642 collectDeclarations(M);2643 2644 // Number rest of registers from N+1 onwards.2645 numberRegistersGlobally(M);2646 2647 // Collect OpName, OpEntryPoint, OpDecorate etc, process other instructions.2648 processOtherInstrs(M);2649 2650 // If there are no entry points, we need the Linkage capability.2651 if (MAI.MS[SPIRV::MB_EntryPoints].empty())2652 MAI.Reqs.addCapability(SPIRV::Capability::Linkage);2653 2654 // Set maximum ID used.2655 GR->setBound(MAI.MaxID);2656 2657 return false;2658}2659