brintos / llvm-project-archived public Read only

1//===- MachineVerifier.cpp - Machine Code Verifier ------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Pass to verify generated machine code. The following is checked:10//11// Operand counts: All explicit operands must be present.12//13// Register classes: All physical and virtual register operands must be14// compatible with the register class required by the instruction descriptor.15//16// Register live intervals: Registers must be defined only once, and must be17// defined before use.18//19// The machine code verifier is enabled with the command-line option20// -verify-machineinstrs.21//===----------------------------------------------------------------------===//22 23#include "llvm/CodeGen/MachineVerifier.h"24#include "llvm/ADT/BitVector.h"25#include "llvm/ADT/DenseMap.h"26#include "llvm/ADT/DenseSet.h"27#include "llvm/ADT/DepthFirstIterator.h"28#include "llvm/ADT/PostOrderIterator.h"29#include "llvm/ADT/STLExtras.h"30#include "llvm/ADT/SetOperations.h"31#include "llvm/ADT/SmallPtrSet.h"32#include "llvm/ADT/SmallVector.h"33#include "llvm/ADT/StringRef.h"34#include "llvm/ADT/Twine.h"35#include "llvm/CodeGen/CodeGenCommonISel.h"36#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"37#include "llvm/CodeGen/LiveInterval.h"38#include "llvm/CodeGen/LiveIntervals.h"39#include "llvm/CodeGen/LiveRangeCalc.h"40#include "llvm/CodeGen/LiveStacks.h"41#include "llvm/CodeGen/LiveVariables.h"42#include "llvm/CodeGen/MachineBasicBlock.h"43#include "llvm/CodeGen/MachineConvergenceVerifier.h"44#include "llvm/CodeGen/MachineDominators.h"45#include "llvm/CodeGen/MachineFrameInfo.h"46#include "llvm/CodeGen/MachineFunction.h"47#include "llvm/CodeGen/MachineFunctionPass.h"48#include "llvm/CodeGen/MachineInstr.h"49#include "llvm/CodeGen/MachineInstrBundle.h"50#include "llvm/CodeGen/MachineMemOperand.h"51#include "llvm/CodeGen/MachineOperand.h"52#include "llvm/CodeGen/MachineRegisterInfo.h"53#include "llvm/CodeGen/PseudoSourceValue.h"54#include "llvm/CodeGen/RegisterBank.h"55#include "llvm/CodeGen/RegisterBankInfo.h"56#include "llvm/CodeGen/SlotIndexes.h"57#include "llvm/CodeGen/StackMaps.h"58#include "llvm/CodeGen/TargetInstrInfo.h"59#include "llvm/CodeGen/TargetLowering.h"60#include "llvm/CodeGen/TargetOpcodes.h"61#include "llvm/CodeGen/TargetRegisterInfo.h"62#include "llvm/CodeGen/TargetSubtargetInfo.h"63#include "llvm/CodeGenTypes/LowLevelType.h"64#include "llvm/IR/BasicBlock.h"65#include "llvm/IR/Constants.h"66#include "llvm/IR/EHPersonalities.h"67#include "llvm/IR/Function.h"68#include "llvm/IR/InlineAsm.h"69#include "llvm/IR/Instructions.h"70#include "llvm/InitializePasses.h"71#include "llvm/MC/LaneBitmask.h"72#include "llvm/MC/MCAsmInfo.h"73#include "llvm/MC/MCDwarf.h"74#include "llvm/MC/MCInstrDesc.h"75#include "llvm/MC/MCRegisterInfo.h"76#include "llvm/MC/MCTargetOptions.h"77#include "llvm/Pass.h"78#include "llvm/Support/Casting.h"79#include "llvm/Support/ErrorHandling.h"80#include "llvm/Support/ManagedStatic.h"81#include "llvm/Support/MathExtras.h"82#include "llvm/Support/ModRef.h"83#include "llvm/Support/Mutex.h"84#include "llvm/Support/raw_ostream.h"85#include "llvm/Target/TargetMachine.h"86#include <algorithm>87#include <cassert>88#include <cstddef>89#include <cstdint>90#include <iterator>91#include <string>92#include <utility>93 94using namespace llvm;95 96namespace {97 98/// Used the by the ReportedErrors class to guarantee only one error is reported99/// at one time.100static ManagedStatic<sys::SmartMutex<true>> ReportedErrorsLock;101 102struct MachineVerifier {103  MachineVerifier(MachineFunctionAnalysisManager &MFAM, const char *b,104                  raw_ostream *OS, bool AbortOnError = true)105      : MFAM(&MFAM), OS(OS ? *OS : nulls()), Banner(b),106        ReportedErrs(AbortOnError) {}107 108  MachineVerifier(Pass *pass, const char *b, raw_ostream *OS,109                  bool AbortOnError = true)110      : PASS(pass), OS(OS ? *OS : nulls()), Banner(b),111        ReportedErrs(AbortOnError) {}112 113  MachineVerifier(const char *b, LiveVariables *LiveVars,114                  LiveIntervals *LiveInts, LiveStacks *LiveStks,115                  SlotIndexes *Indexes, raw_ostream *OS,116                  bool AbortOnError = true)117      : OS(OS ? *OS : nulls()), Banner(b), LiveVars(LiveVars),118        LiveInts(LiveInts), LiveStks(LiveStks), Indexes(Indexes),119        ReportedErrs(AbortOnError) {}120 121  /// \returns true if no problems were found.122  bool verify(const MachineFunction &MF);123 124  MachineFunctionAnalysisManager *MFAM = nullptr;125  Pass *const PASS = nullptr;126  raw_ostream &OS;127  const char *Banner;128  const MachineFunction *MF = nullptr;129  const TargetMachine *TM = nullptr;130  const TargetInstrInfo *TII = nullptr;131  const TargetRegisterInfo *TRI = nullptr;132  const MachineRegisterInfo *MRI = nullptr;133  const RegisterBankInfo *RBI = nullptr;134 135  // Avoid querying the MachineFunctionProperties for each operand.136  bool isFunctionRegBankSelected = false;137  bool isFunctionSelected = false;138  bool isFunctionTracksDebugUserValues = false;139 140  using RegVector = SmallVector<Register, 16>;141  using RegMaskVector = SmallVector<const uint32_t *, 4>;142  using RegSet = DenseSet<Register>;143  using RegMap = DenseMap<Register, const MachineInstr *>;144  using BlockSet = SmallPtrSet<const MachineBasicBlock *, 8>;145 146  const MachineInstr *FirstNonPHI = nullptr;147  const MachineInstr *FirstTerminator = nullptr;148  BlockSet FunctionBlocks;149 150  BitVector regsReserved;151  RegSet regsLive;152  RegVector regsDefined, regsDead, regsKilled;153  RegMaskVector regMasks;154 155  SlotIndex lastIndex;156 157  // Add Reg and any sub-registers to RV158  void addRegWithSubRegs(RegVector &RV, Register Reg) {159    RV.push_back(Reg);160    if (Reg.isPhysical())161      append_range(RV, TRI->subregs(Reg.asMCReg()));162  }163 164  struct BBInfo {165    // Is this MBB reachable from the MF entry point?166    bool reachable = false;167 168    // Vregs that must be live in because they are used without being169    // defined. Map value is the user. vregsLiveIn doesn't include regs170    // that only are used by PHI nodes.171    RegMap vregsLiveIn;172 173    // Regs killed in MBB. They may be defined again, and will then be in both174    // regsKilled and regsLiveOut.175    RegSet regsKilled;176 177    // Regs defined in MBB and live out. Note that vregs passing through may178    // be live out without being mentioned here.179    RegSet regsLiveOut;180 181    // Vregs that pass through MBB untouched. This set is disjoint from182    // regsKilled and regsLiveOut.183    RegSet vregsPassed;184 185    // Vregs that must pass through MBB because they are needed by a successor186    // block. This set is disjoint from regsLiveOut.187    RegSet vregsRequired;188 189    // Set versions of block's predecessor and successor lists.190    BlockSet Preds, Succs;191 192    BBInfo() = default;193 194    // Add register to vregsRequired if it belongs there. Return true if195    // anything changed.196    bool addRequired(Register Reg) {197      if (!Reg.isVirtual())198        return false;199      if (regsLiveOut.count(Reg))200        return false;201      return vregsRequired.insert(Reg).second;202    }203 204    // Same for a full set.205    bool addRequired(const RegSet &RS) {206      bool Changed = false;207      for (Register Reg : RS)208        Changed |= addRequired(Reg);209      return Changed;210    }211 212    // Same for a full map.213    bool addRequired(const RegMap &RM) {214      bool Changed = false;215      for (const auto &I : RM)216        Changed |= addRequired(I.first);217      return Changed;218    }219 220    // Live-out registers are either in regsLiveOut or vregsPassed.221    bool isLiveOut(Register Reg) const {222      return regsLiveOut.count(Reg) || vregsPassed.count(Reg);223    }224  };225 226  // Extra register info per MBB.227  DenseMap<const MachineBasicBlock *, BBInfo> MBBInfoMap;228 229  bool isReserved(Register Reg) {230    return Reg.id() < regsReserved.size() && regsReserved.test(Reg.id());231  }232 233  bool isAllocatable(Register Reg) const {234    return Reg.id() < TRI->getNumRegs() && TRI->isInAllocatableClass(Reg) &&235           !regsReserved.test(Reg.id());236  }237 238  // Analysis information if available239  LiveVariables *LiveVars = nullptr;240  LiveIntervals *LiveInts = nullptr;241  LiveStacks *LiveStks = nullptr;242  SlotIndexes *Indexes = nullptr;243 244  /// A class to track the number of reported error and to guarantee that only245  /// one error is reported at one time.246  class ReportedErrors {247    unsigned NumReported = 0;248    bool AbortOnError;249 250  public:251    /// \param AbortOnError -- If set, abort after printing the first error.252    ReportedErrors(bool AbortOnError) : AbortOnError(AbortOnError) {}253 254    ~ReportedErrors() {255      if (!hasError())256        return;257      if (AbortOnError)258        report_fatal_error("Found " + Twine(NumReported) +259                           " machine code errors.");260      // Since we haven't aborted, release the lock to allow other threads to261      // report errors.262      ReportedErrorsLock->unlock();263    }264 265    /// Increment the number of reported errors.266    /// \returns true if this is the first reported error.267    bool increment() {268      // If this is the first error this thread has encountered, grab the lock269      // to prevent other threads from reporting errors at the same time.270      // Otherwise we assume we already have the lock.271      if (!hasError())272        ReportedErrorsLock->lock();273      ++NumReported;274      return NumReported == 1;275    }276 277    /// \returns true if an error was reported.278    bool hasError() { return NumReported; }279  };280  ReportedErrors ReportedErrs;281 282  // This is calculated only when trying to verify convergence control tokens.283  // Similar to the LLVM IR verifier, we calculate this locally instead of284  // relying on the pass manager.285  MachineDominatorTree DT;286 287  void visitMachineFunctionBefore();288  void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB);289  void visitMachineBundleBefore(const MachineInstr *MI);290 291  /// Verify that all of \p MI's virtual register operands are scalars.292  /// \returns True if all virtual register operands are scalar. False293  /// otherwise.294  bool verifyAllRegOpsScalar(const MachineInstr &MI,295                             const MachineRegisterInfo &MRI);296  bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI);297 298  bool verifyGIntrinsicSideEffects(const MachineInstr *MI);299  bool verifyGIntrinsicConvergence(const MachineInstr *MI);300  void verifyPreISelGenericInstruction(const MachineInstr *MI);301 302  void visitMachineInstrBefore(const MachineInstr *MI);303  void visitMachineOperand(const MachineOperand *MO, unsigned MONum);304  void visitMachineBundleAfter(const MachineInstr *MI);305  void visitMachineBasicBlockAfter(const MachineBasicBlock *MBB);306  void visitMachineFunctionAfter();307 308  void report(const char *msg, const MachineFunction *MF);309  void report(const char *msg, const MachineBasicBlock *MBB);310  void report(const char *msg, const MachineInstr *MI);311  void report(const char *msg, const MachineOperand *MO, unsigned MONum,312              LLT MOVRegType = LLT{});313  void report(const Twine &Msg, const MachineInstr *MI);314 315  void report_context(const LiveInterval &LI) const;316  void report_context(const LiveRange &LR, VirtRegOrUnit VRegOrUnit,317                      LaneBitmask LaneMask) const;318  void report_context(const LiveRange::Segment &S) const;319  void report_context(const VNInfo &VNI) const;320  void report_context(SlotIndex Pos) const;321  void report_context(MCPhysReg PhysReg) const;322  void report_context_liverange(const LiveRange &LR) const;323  void report_context_lanemask(LaneBitmask LaneMask) const;324  void report_context_vreg(Register VReg) const;325  void report_context_vreg_regunit(VirtRegOrUnit VRegOrUnit) const;326 327  void verifyInlineAsm(const MachineInstr *MI);328 329  void checkLiveness(const MachineOperand *MO, unsigned MONum);330  void checkLivenessAtUse(const MachineOperand *MO, unsigned MONum,331                          SlotIndex UseIdx, const LiveRange &LR,332                          VirtRegOrUnit VRegOrUnit,333                          LaneBitmask LaneMask = LaneBitmask::getNone());334  void checkLivenessAtDef(const MachineOperand *MO, unsigned MONum,335                          SlotIndex DefIdx, const LiveRange &LR,336                          VirtRegOrUnit VRegOrUnit, bool SubRangeCheck = false,337                          LaneBitmask LaneMask = LaneBitmask::getNone());338 339  void markReachable(const MachineBasicBlock *MBB);340  void calcRegsPassed();341  void checkPHIOps(const MachineBasicBlock &MBB);342 343  void calcRegsRequired();344  void verifyLiveVariables();345  void verifyLiveIntervals();346  void verifyLiveInterval(const LiveInterval &);347  void verifyLiveRangeValue(const LiveRange &, const VNInfo *, VirtRegOrUnit,348                            LaneBitmask);349  void verifyLiveRangeSegment(const LiveRange &,350                              const LiveRange::const_iterator I, VirtRegOrUnit,351                              LaneBitmask);352  void verifyLiveRange(const LiveRange &, VirtRegOrUnit,353                       LaneBitmask LaneMask = LaneBitmask::getNone());354 355  void verifyStackFrame();356  /// Check that the stack protector is the top-most object in the stack.357  void verifyStackProtector();358 359  void verifySlotIndexes() const;360  void verifyProperties(const MachineFunction &MF);361};362 363struct MachineVerifierLegacyPass : public MachineFunctionPass {364  static char ID; // Pass ID, replacement for typeid365 366  const std::string Banner;367 368  MachineVerifierLegacyPass(std::string banner = std::string())369      : MachineFunctionPass(ID), Banner(std::move(banner)) {370    initializeMachineVerifierLegacyPassPass(*PassRegistry::getPassRegistry());371  }372 373  void getAnalysisUsage(AnalysisUsage &AU) const override {374    AU.addUsedIfAvailable<LiveStacksWrapperLegacy>();375    AU.addUsedIfAvailable<LiveVariablesWrapperPass>();376    AU.addUsedIfAvailable<SlotIndexesWrapperPass>();377    AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();378    AU.setPreservesAll();379    MachineFunctionPass::getAnalysisUsage(AU);380  }381 382  bool runOnMachineFunction(MachineFunction &MF) override {383    // Skip functions that have known verification problems.384    // FIXME: Remove this mechanism when all problematic passes have been385    // fixed.386    if (MF.getProperties().hasFailsVerification())387      return false;388 389    MachineVerifier(this, Banner.c_str(), &errs()).verify(MF);390    return false;391  }392};393 394} // end anonymous namespace395 396PreservedAnalyses397MachineVerifierPass::run(MachineFunction &MF,398                         MachineFunctionAnalysisManager &MFAM) {399  // Skip functions that have known verification problems.400  // FIXME: Remove this mechanism when all problematic passes have been401  // fixed.402  if (MF.getProperties().hasFailsVerification())403    return PreservedAnalyses::all();404  MachineVerifier(MFAM, Banner.c_str(), &errs()).verify(MF);405  return PreservedAnalyses::all();406}407 408char MachineVerifierLegacyPass::ID = 0;409 410INITIALIZE_PASS(MachineVerifierLegacyPass, "machineverifier",411                "Verify generated machine code", false, false)412 413FunctionPass *llvm::createMachineVerifierPass(const std::string &Banner) {414  return new MachineVerifierLegacyPass(Banner);415}416 417void llvm::verifyMachineFunction(const std::string &Banner,418                                 const MachineFunction &MF) {419  // TODO: Use MFAM after porting below analyses.420  // LiveVariables *LiveVars;421  // LiveIntervals *LiveInts;422  // LiveStacks *LiveStks;423  // SlotIndexes *Indexes;424  MachineVerifier(nullptr, Banner.c_str(), &errs()).verify(MF);425}426 427bool MachineFunction::verify(Pass *p, const char *Banner, raw_ostream *OS,428                             bool AbortOnError) const {429  return MachineVerifier(p, Banner, OS, AbortOnError).verify(*this);430}431 432bool MachineFunction::verify(MachineFunctionAnalysisManager &MFAM,433                             const char *Banner, raw_ostream *OS,434                             bool AbortOnError) const {435  return MachineVerifier(MFAM, Banner, OS, AbortOnError).verify(*this);436}437 438bool MachineFunction::verify(LiveIntervals *LiveInts, SlotIndexes *Indexes,439                             const char *Banner, raw_ostream *OS,440                             bool AbortOnError) const {441  return MachineVerifier(Banner, /*LiveVars=*/nullptr, LiveInts,442                         /*LiveStks=*/nullptr, Indexes, OS, AbortOnError)443      .verify(*this);444}445 446void MachineVerifier::verifySlotIndexes() const {447  if (Indexes == nullptr)448    return;449 450  // Ensure the IdxMBB list is sorted by slot indexes.451  SlotIndex Last;452  for (SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin(),453       E = Indexes->MBBIndexEnd(); I != E; ++I) {454    assert(!Last.isValid() || I->first > Last);455    Last = I->first;456  }457}458 459void MachineVerifier::verifyProperties(const MachineFunction &MF) {460  // If a pass has introduced virtual registers without clearing the461  // NoVRegs property (or set it without allocating the vregs)462  // then report an error.463  if (MF.getProperties().hasNoVRegs() && MRI->getNumVirtRegs())464    report("Function has NoVRegs property but there are VReg operands", &MF);465}466 467bool MachineVerifier::verify(const MachineFunction &MF) {468  this->MF = &MF;469  TM = &MF.getTarget();470  TII = MF.getSubtarget().getInstrInfo();471  TRI = MF.getSubtarget().getRegisterInfo();472  RBI = MF.getSubtarget().getRegBankInfo();473  MRI = &MF.getRegInfo();474 475  const MachineFunctionProperties &Props = MF.getProperties();476  const bool isFunctionFailedISel = Props.hasFailedISel();477 478  // If we're mid-GlobalISel and we already triggered the fallback path then479  // it's expected that the MIR is somewhat broken but that's ok since we'll480  // reset it and clear the FailedISel attribute in ResetMachineFunctions.481  if (isFunctionFailedISel)482    return true;483 484  isFunctionRegBankSelected = Props.hasRegBankSelected();485  isFunctionSelected = Props.hasSelected();486  isFunctionTracksDebugUserValues = Props.hasTracksDebugUserValues();487 488  if (PASS) {489    auto *LISWrapper = PASS->getAnalysisIfAvailable<LiveIntervalsWrapperPass>();490    LiveInts = LISWrapper ? &LISWrapper->getLIS() : nullptr;491    // We don't want to verify LiveVariables if LiveIntervals is available.492    auto *LVWrapper = PASS->getAnalysisIfAvailable<LiveVariablesWrapperPass>();493    if (!LiveInts)494      LiveVars = LVWrapper ? &LVWrapper->getLV() : nullptr;495    auto *LSWrapper = PASS->getAnalysisIfAvailable<LiveStacksWrapperLegacy>();496    LiveStks = LSWrapper ? &LSWrapper->getLS() : nullptr;497    auto *SIWrapper = PASS->getAnalysisIfAvailable<SlotIndexesWrapperPass>();498    Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;499  }500  if (MFAM) {501    MachineFunction &Func = const_cast<MachineFunction &>(MF);502    LiveInts = MFAM->getCachedResult<LiveIntervalsAnalysis>(Func);503    if (!LiveInts)504      LiveVars = MFAM->getCachedResult<LiveVariablesAnalysis>(Func);505    // TODO: LiveStks = MFAM->getCachedResult<LiveStacksAnalysis>(Func);506    Indexes = MFAM->getCachedResult<SlotIndexesAnalysis>(Func);507  }508 509  verifySlotIndexes();510 511  verifyProperties(MF);512 513  visitMachineFunctionBefore();514  for (const MachineBasicBlock &MBB : MF) {515    visitMachineBasicBlockBefore(&MBB);516    // Keep track of the current bundle header.517    const MachineInstr *CurBundle = nullptr;518    // Do we expect the next instruction to be part of the same bundle?519    bool InBundle = false;520 521    for (const MachineInstr &MI : MBB.instrs()) {522      if (MI.getParent() != &MBB) {523        report("Bad instruction parent pointer", &MBB);524        OS << "Instruction: " << MI;525        continue;526      }527 528      // Check for consistent bundle flags.529      if (InBundle && !MI.isBundledWithPred())530        report("Missing BundledPred flag, "531               "BundledSucc was set on predecessor",532               &MI);533      if (!InBundle && MI.isBundledWithPred())534        report("BundledPred flag is set, "535               "but BundledSucc not set on predecessor",536               &MI);537 538      // Is this a bundle header?539      if (!MI.isInsideBundle()) {540        if (CurBundle)541          visitMachineBundleAfter(CurBundle);542        CurBundle = &MI;543        visitMachineBundleBefore(CurBundle);544      } else if (!CurBundle)545        report("No bundle header", &MI);546      visitMachineInstrBefore(&MI);547      for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {548        const MachineOperand &Op = MI.getOperand(I);549        if (Op.getParent() != &MI) {550          // Make sure to use correct addOperand / removeOperand / ChangeTo551          // functions when replacing operands of a MachineInstr.552          report("Instruction has operand with wrong parent set", &MI);553        }554 555        visitMachineOperand(&Op, I);556      }557 558      // Was this the last bundled instruction?559      InBundle = MI.isBundledWithSucc();560    }561    if (CurBundle)562      visitMachineBundleAfter(CurBundle);563    if (InBundle)564      report("BundledSucc flag set on last instruction in block", &MBB.back());565    visitMachineBasicBlockAfter(&MBB);566  }567  visitMachineFunctionAfter();568 569  // Clean up.570  regsLive.clear();571  regsDefined.clear();572  regsDead.clear();573  regsKilled.clear();574  regMasks.clear();575  MBBInfoMap.clear();576 577  return !ReportedErrs.hasError();578}579 580void MachineVerifier::report(const char *msg, const MachineFunction *MF) {581  assert(MF);582  OS << '\n';583  if (ReportedErrs.increment()) {584    if (Banner)585      OS << "# " << Banner << '\n';586 587    if (LiveInts != nullptr)588      LiveInts->print(OS);589    else590      MF->print(OS, Indexes);591  }592 593  OS << "*** Bad machine code: " << msg << " ***\n"594     << "- function:    " << MF->getName() << '\n';595}596 597void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) {598  assert(MBB);599  report(msg, MBB->getParent());600  OS << "- basic block: " << printMBBReference(*MBB) << ' ' << MBB->getName()601     << " (" << (const void *)MBB << ')';602  if (Indexes)603    OS << " [" << Indexes->getMBBStartIdx(MBB) << ';'604       << Indexes->getMBBEndIdx(MBB) << ')';605  OS << '\n';606}607 608void MachineVerifier::report(const char *msg, const MachineInstr *MI) {609  assert(MI);610  report(msg, MI->getParent());611  OS << "- instruction: ";612  if (Indexes && Indexes->hasIndex(*MI))613    OS << Indexes->getInstructionIndex(*MI) << '\t';614  MI->print(OS, /*IsStandalone=*/true);615}616 617void MachineVerifier::report(const char *msg, const MachineOperand *MO,618                             unsigned MONum, LLT MOVRegType) {619  assert(MO);620  report(msg, MO->getParent());621  OS << "- operand " << MONum << ":   ";622  MO->print(OS, MOVRegType, TRI);623  OS << '\n';624}625 626void MachineVerifier::report(const Twine &Msg, const MachineInstr *MI) {627  report(Msg.str().c_str(), MI);628}629 630void MachineVerifier::report_context(SlotIndex Pos) const {631  OS << "- at:          " << Pos << '\n';632}633 634void MachineVerifier::report_context(const LiveInterval &LI) const {635  OS << "- interval:    " << LI << '\n';636}637 638void MachineVerifier::report_context(const LiveRange &LR,639                                     VirtRegOrUnit VRegOrUnit,640                                     LaneBitmask LaneMask) const {641  report_context_liverange(LR);642  report_context_vreg_regunit(VRegOrUnit);643  if (LaneMask.any())644    report_context_lanemask(LaneMask);645}646 647void MachineVerifier::report_context(const LiveRange::Segment &S) const {648  OS << "- segment:     " << S << '\n';649}650 651void MachineVerifier::report_context(const VNInfo &VNI) const {652  OS << "- ValNo:       " << VNI.id << " (def " << VNI.def << ")\n";653}654 655void MachineVerifier::report_context_liverange(const LiveRange &LR) const {656  OS << "- liverange:   " << LR << '\n';657}658 659void MachineVerifier::report_context(MCPhysReg PReg) const {660  OS << "- p. register: " << printReg(PReg, TRI) << '\n';661}662 663void MachineVerifier::report_context_vreg(Register VReg) const {664  OS << "- v. register: " << printReg(VReg, TRI) << '\n';665}666 667void MachineVerifier::report_context_vreg_regunit(668    VirtRegOrUnit VRegOrUnit) const {669  if (VRegOrUnit.isVirtualReg()) {670    report_context_vreg(VRegOrUnit.asVirtualReg());671  } else {672    OS << "- regunit:     " << printRegUnit(VRegOrUnit.asMCRegUnit(), TRI)673       << '\n';674  }675}676 677void MachineVerifier::report_context_lanemask(LaneBitmask LaneMask) const {678  OS << "- lanemask:    " << PrintLaneMask(LaneMask) << '\n';679}680 681void MachineVerifier::markReachable(const MachineBasicBlock *MBB) {682  BBInfo &MInfo = MBBInfoMap[MBB];683  if (!MInfo.reachable) {684    MInfo.reachable = true;685    for (const MachineBasicBlock *Succ : MBB->successors())686      markReachable(Succ);687  }688}689 690void MachineVerifier::visitMachineFunctionBefore() {691  lastIndex = SlotIndex();692  regsReserved = MRI->reservedRegsFrozen() ? MRI->getReservedRegs()693                                           : TRI->getReservedRegs(*MF);694 695  if (!MF->empty())696    markReachable(&MF->front());697 698  // Build a set of the basic blocks in the function.699  FunctionBlocks.clear();700  for (const auto &MBB : *MF) {701    FunctionBlocks.insert(&MBB);702    BBInfo &MInfo = MBBInfoMap[&MBB];703 704    MInfo.Preds.insert_range(MBB.predecessors());705    if (MInfo.Preds.size() != MBB.pred_size())706      report("MBB has duplicate entries in its predecessor list.", &MBB);707 708    MInfo.Succs.insert_range(MBB.successors());709    if (MInfo.Succs.size() != MBB.succ_size())710      report("MBB has duplicate entries in its successor list.", &MBB);711  }712 713  // Check that the register use lists are sane.714  MRI->verifyUseLists();715 716  if (!MF->empty()) {717    verifyStackFrame();718    verifyStackProtector();719  }720}721 722void723MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {724  FirstTerminator = nullptr;725  FirstNonPHI = nullptr;726 727  if (!MF->getProperties().hasNoPHIs() && MRI->tracksLiveness()) {728    // If this block has allocatable physical registers live-in, check that729    // it is an entry block or landing pad.730    for (const auto &LI : MBB->liveins()) {731      if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() &&732          MBB->getIterator() != MBB->getParent()->begin() &&733          !MBB->isInlineAsmBrIndirectTarget()) {734        report("MBB has allocatable live-in, but isn't entry, landing-pad, or "735               "inlineasm-br-indirect-target.",736               MBB);737        report_context(LI.PhysReg);738      }739    }740  }741 742  if (MBB->isIRBlockAddressTaken()) {743    if (!MBB->getAddressTakenIRBlock()->hasAddressTaken())744      report("ir-block-address-taken is associated with basic block not used by "745             "a blockaddress.",746             MBB);747  }748 749  // Count the number of landing pad successors.750  SmallPtrSet<const MachineBasicBlock*, 4> LandingPadSuccs;751  for (const auto *succ : MBB->successors()) {752    if (succ->isEHPad())753      LandingPadSuccs.insert(succ);754    if (!FunctionBlocks.count(succ))755      report("MBB has successor that isn't part of the function.", MBB);756    if (!MBBInfoMap[succ].Preds.count(MBB)) {757      report("Inconsistent CFG", MBB);758      OS << "MBB is not in the predecessor list of the successor "759         << printMBBReference(*succ) << ".\n";760    }761  }762 763  // Check the predecessor list.764  for (const MachineBasicBlock *Pred : MBB->predecessors()) {765    if (!FunctionBlocks.count(Pred))766      report("MBB has predecessor that isn't part of the function.", MBB);767    if (!MBBInfoMap[Pred].Succs.count(MBB)) {768      report("Inconsistent CFG", MBB);769      OS << "MBB is not in the successor list of the predecessor "770         << printMBBReference(*Pred) << ".\n";771    }772  }773 774  const MCAsmInfo *AsmInfo = TM->getMCAsmInfo();775  const BasicBlock *BB = MBB->getBasicBlock();776  const Function &F = MF->getFunction();777  if (LandingPadSuccs.size() > 1 &&778      !(AsmInfo &&779        AsmInfo->getExceptionHandlingType() == ExceptionHandling::SjLj &&780        BB && isa<SwitchInst>(BB->getTerminator())) &&781      !isScopedEHPersonality(classifyEHPersonality(F.getPersonalityFn())))782    report("MBB has more than one landing pad successor", MBB);783 784  // Call analyzeBranch. If it succeeds, there several more conditions to check.785  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;786  SmallVector<MachineOperand, 4> Cond;787  if (!TII->analyzeBranch(*const_cast<MachineBasicBlock *>(MBB), TBB, FBB,788                          Cond)) {789    // Ok, analyzeBranch thinks it knows what's going on with this block. Let's790    // check whether its answers match up with reality.791    if (!TBB && !FBB) {792      // Block falls through to its successor.793      if (!MBB->empty() && MBB->back().isBarrier() &&794          !TII->isPredicated(MBB->back())) {795        report("MBB exits via unconditional fall-through but ends with a "796               "barrier instruction!", MBB);797      }798      if (!Cond.empty()) {799        report("MBB exits via unconditional fall-through but has a condition!",800               MBB);801      }802    } else if (TBB && !FBB && Cond.empty()) {803      // Block unconditionally branches somewhere.804      if (MBB->empty()) {805        report("MBB exits via unconditional branch but doesn't contain "806               "any instructions!", MBB);807      } else if (!MBB->back().isBarrier()) {808        report("MBB exits via unconditional branch but doesn't end with a "809               "barrier instruction!", MBB);810      } else if (!MBB->back().isTerminator()) {811        report("MBB exits via unconditional branch but the branch isn't a "812               "terminator instruction!", MBB);813      }814    } else if (TBB && !FBB && !Cond.empty()) {815      // Block conditionally branches somewhere, otherwise falls through.816      if (MBB->empty()) {817        report("MBB exits via conditional branch/fall-through but doesn't "818               "contain any instructions!", MBB);819      } else if (MBB->back().isBarrier()) {820        report("MBB exits via conditional branch/fall-through but ends with a "821               "barrier instruction!", MBB);822      } else if (!MBB->back().isTerminator()) {823        report("MBB exits via conditional branch/fall-through but the branch "824               "isn't a terminator instruction!", MBB);825      }826    } else if (TBB && FBB) {827      // Block conditionally branches somewhere, otherwise branches828      // somewhere else.829      if (MBB->empty()) {830        report("MBB exits via conditional branch/branch but doesn't "831               "contain any instructions!", MBB);832      } else if (!MBB->back().isBarrier()) {833        report("MBB exits via conditional branch/branch but doesn't end with a "834               "barrier instruction!", MBB);835      } else if (!MBB->back().isTerminator()) {836        report("MBB exits via conditional branch/branch but the branch "837               "isn't a terminator instruction!", MBB);838      }839      if (Cond.empty()) {840        report("MBB exits via conditional branch/branch but there's no "841               "condition!", MBB);842      }843    } else {844      report("analyzeBranch returned invalid data!", MBB);845    }846 847    // Now check that the successors match up with the answers reported by848    // analyzeBranch.849    if (TBB && !MBB->isSuccessor(TBB))850      report("MBB exits via jump or conditional branch, but its target isn't a "851             "CFG successor!",852             MBB);853    if (FBB && !MBB->isSuccessor(FBB))854      report("MBB exits via conditional branch, but its target isn't a CFG "855             "successor!",856             MBB);857 858    // There might be a fallthrough to the next block if there's either no859    // unconditional true branch, or if there's a condition, and one of the860    // branches is missing.861    bool Fallthrough = !TBB || (!Cond.empty() && !FBB);862 863    // A conditional fallthrough must be an actual CFG successor, not864    // unreachable. (Conversely, an unconditional fallthrough might not really865    // be a successor, because the block might end in unreachable.)866    if (!Cond.empty() && !FBB) {867      MachineFunction::const_iterator MBBI = std::next(MBB->getIterator());868      if (MBBI == MF->end()) {869        report("MBB conditionally falls through out of function!", MBB);870      } else if (!MBB->isSuccessor(&*MBBI))871        report("MBB exits via conditional branch/fall-through but the CFG "872               "successors don't match the actual successors!",873               MBB);874    }875 876    // Verify that there aren't any extra un-accounted-for successors.877    for (const MachineBasicBlock *SuccMBB : MBB->successors()) {878      // If this successor is one of the branch targets, it's okay.879      if (SuccMBB == TBB || SuccMBB == FBB)880        continue;881      // If we might have a fallthrough, and the successor is the fallthrough882      // block, that's also ok.883      if (Fallthrough && SuccMBB == MBB->getNextNode())884        continue;885      // Also accept successors which are for exception-handling or might be886      // inlineasm_br targets.887      if (SuccMBB->isEHPad() || SuccMBB->isInlineAsmBrIndirectTarget())888        continue;889      report("MBB has unexpected successors which are not branch targets, "890             "fallthrough, EHPads, or inlineasm_br targets.",891             MBB);892    }893  }894 895  regsLive.clear();896  if (MRI->tracksLiveness()) {897    for (const auto &LI : MBB->liveins()) {898      if (!LI.PhysReg.isPhysical()) {899        report("MBB live-in list contains non-physical register", MBB);900        continue;901      }902      regsLive.insert_range(TRI->subregs_inclusive(LI.PhysReg));903    }904  }905 906  const MachineFrameInfo &MFI = MF->getFrameInfo();907  BitVector PR = MFI.getPristineRegs(*MF);908  for (unsigned I : PR.set_bits())909    regsLive.insert_range(TRI->subregs_inclusive(I));910 911  regsKilled.clear();912  regsDefined.clear();913 914  if (Indexes)915    lastIndex = Indexes->getMBBStartIdx(MBB);916}917 918// This function gets called for all bundle headers, including normal919// stand-alone unbundled instructions.920void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) {921  if (Indexes && Indexes->hasIndex(*MI)) {922    SlotIndex idx = Indexes->getInstructionIndex(*MI);923    if (!(idx > lastIndex)) {924      report("Instruction index out of order", MI);925      OS << "Last instruction was at " << lastIndex << '\n';926    }927    lastIndex = idx;928  }929 930  // Ensure non-terminators don't follow terminators.931  if (MI->isTerminator()) {932    if (!FirstTerminator)933      FirstTerminator = MI;934  } else if (FirstTerminator) {935    // For GlobalISel, G_INVOKE_REGION_START is a terminator that we allow to936    // precede non-terminators.937    if (FirstTerminator->getOpcode() != TargetOpcode::G_INVOKE_REGION_START) {938      report("Non-terminator instruction after the first terminator", MI);939      OS << "First terminator was:\t" << *FirstTerminator;940    }941  }942}943 944// The operands on an INLINEASM instruction must follow a template.945// Verify that the flag operands make sense.946void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) {947  // The first two operands on INLINEASM are the asm string and global flags.948  if (MI->getNumOperands() < 2) {949    report("Too few operands on inline asm", MI);950    return;951  }952  if (!MI->getOperand(0).isSymbol())953    report("Asm string must be an external symbol", MI);954  if (!MI->getOperand(1).isImm())955    report("Asm flags must be an immediate", MI);956  // Allowed flags are Extra_HasSideEffects = 1, Extra_IsAlignStack = 2,957  // Extra_AsmDialect = 4, Extra_MayLoad = 8, and Extra_MayStore = 16,958  // and Extra_IsConvergent = 32.959  if (!isUInt<6>(MI->getOperand(1).getImm()))960    report("Unknown asm flags", &MI->getOperand(1), 1);961 962  static_assert(InlineAsm::MIOp_FirstOperand == 2, "Asm format changed");963 964  unsigned OpNo = InlineAsm::MIOp_FirstOperand;965  unsigned NumOps;966  for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) {967    const MachineOperand &MO = MI->getOperand(OpNo);968    // There may be implicit ops after the fixed operands.969    if (!MO.isImm())970      break;971    const InlineAsm::Flag F(MO.getImm());972    NumOps = 1 + F.getNumOperandRegisters();973  }974 975  if (OpNo > MI->getNumOperands())976    report("Missing operands in last group", MI);977 978  // An optional MDNode follows the groups.979  if (OpNo < MI->getNumOperands() && MI->getOperand(OpNo).isMetadata())980    ++OpNo;981 982  // All trailing operands must be implicit registers.983  for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) {984    const MachineOperand &MO = MI->getOperand(OpNo);985    if (!MO.isReg() || !MO.isImplicit())986      report("Expected implicit register after groups", &MO, OpNo);987  }988 989  if (MI->getOpcode() == TargetOpcode::INLINEASM_BR) {990    const MachineBasicBlock *MBB = MI->getParent();991 992    for (unsigned i = InlineAsm::MIOp_FirstOperand, e = MI->getNumOperands();993         i != e; ++i) {994      const MachineOperand &MO = MI->getOperand(i);995 996      if (!MO.isMBB())997        continue;998 999      // Check the successor & predecessor lists look ok, assume they are1000      // not. Find the indirect target without going through the successors.1001      const MachineBasicBlock *IndirectTargetMBB = MO.getMBB();1002      if (!IndirectTargetMBB) {1003        report("INLINEASM_BR indirect target does not exist", &MO, i);1004        break;1005      }1006 1007      if (!MBB->isSuccessor(IndirectTargetMBB))1008        report("INLINEASM_BR indirect target missing from successor list", &MO,1009               i);1010 1011      if (!IndirectTargetMBB->isPredecessor(MBB))1012        report("INLINEASM_BR indirect target predecessor list missing parent",1013               &MO, i);1014    }1015  }1016}1017 1018bool MachineVerifier::verifyAllRegOpsScalar(const MachineInstr &MI,1019                                            const MachineRegisterInfo &MRI) {1020  if (none_of(MI.explicit_operands(), [&MRI](const MachineOperand &Op) {1021        if (!Op.isReg())1022          return false;1023        const auto Reg = Op.getReg();1024        if (Reg.isPhysical())1025          return false;1026        return !MRI.getType(Reg).isScalar();1027      }))1028    return true;1029  report("All register operands must have scalar types", &MI);1030  return false;1031}1032 1033/// Check that types are consistent when two operands need to have the same1034/// number of vector elements.1035/// \return true if the types are valid.1036bool MachineVerifier::verifyVectorElementMatch(LLT Ty0, LLT Ty1,1037                                               const MachineInstr *MI) {1038  if (Ty0.isVector() != Ty1.isVector()) {1039    report("operand types must be all-vector or all-scalar", MI);1040    // Generally we try to report as many issues as possible at once, but in1041    // this case it's not clear what should we be comparing the size of the1042    // scalar with: the size of the whole vector or its lane. Instead of1043    // making an arbitrary choice and emitting not so helpful message, let's1044    // avoid the extra noise and stop here.1045    return false;1046  }1047 1048  if (Ty0.isVector() && Ty0.getElementCount() != Ty1.getElementCount()) {1049    report("operand types must preserve number of vector elements", MI);1050    return false;1051  }1052 1053  return true;1054}1055 1056bool MachineVerifier::verifyGIntrinsicSideEffects(const MachineInstr *MI) {1057  auto Opcode = MI->getOpcode();1058  bool NoSideEffects = Opcode == TargetOpcode::G_INTRINSIC ||1059                       Opcode == TargetOpcode::G_INTRINSIC_CONVERGENT;1060  unsigned IntrID = cast<GIntrinsic>(MI)->getIntrinsicID();1061  if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {1062    AttributeSet Attrs = Intrinsic::getFnAttributes(1063        MF->getFunction().getContext(), static_cast<Intrinsic::ID>(IntrID));1064    bool DeclHasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();1065    if (NoSideEffects && DeclHasSideEffects) {1066      report(Twine(TII->getName(Opcode),1067                   " used with intrinsic that accesses memory"),1068             MI);1069      return false;1070    }1071    if (!NoSideEffects && !DeclHasSideEffects) {1072      report(Twine(TII->getName(Opcode), " used with readnone intrinsic"), MI);1073      return false;1074    }1075  }1076 1077  return true;1078}1079 1080bool MachineVerifier::verifyGIntrinsicConvergence(const MachineInstr *MI) {1081  auto Opcode = MI->getOpcode();1082  bool NotConvergent = Opcode == TargetOpcode::G_INTRINSIC ||1083                       Opcode == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;1084  unsigned IntrID = cast<GIntrinsic>(MI)->getIntrinsicID();1085  if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {1086    AttributeSet Attrs = Intrinsic::getFnAttributes(1087        MF->getFunction().getContext(), static_cast<Intrinsic::ID>(IntrID));1088    bool DeclIsConvergent = Attrs.hasAttribute(Attribute::Convergent);1089    if (NotConvergent && DeclIsConvergent) {1090      report(Twine(TII->getName(Opcode), " used with a convergent intrinsic"),1091             MI);1092      return false;1093    }1094    if (!NotConvergent && !DeclIsConvergent) {1095      report(1096          Twine(TII->getName(Opcode), " used with a non-convergent intrinsic"),1097          MI);1098      return false;1099    }1100  }1101 1102  return true;1103}1104 1105void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {1106  if (isFunctionSelected)1107    report("Unexpected generic instruction in a Selected function", MI);1108 1109  const MCInstrDesc &MCID = MI->getDesc();1110  unsigned NumOps = MI->getNumOperands();1111 1112  // Branches must reference a basic block if they are not indirect1113  if (MI->isBranch() && !MI->isIndirectBranch()) {1114    bool HasMBB = false;1115    for (const MachineOperand &Op : MI->operands()) {1116      if (Op.isMBB()) {1117        HasMBB = true;1118        break;1119      }1120    }1121 1122    if (!HasMBB) {1123      report("Branch instruction is missing a basic block operand or "1124             "isIndirectBranch property",1125             MI);1126    }1127  }1128 1129  // Check types.1130  SmallVector<LLT, 4> Types;1131  for (unsigned I = 0, E = std::min(MCID.getNumOperands(), NumOps);1132       I != E; ++I) {1133    if (!MCID.operands()[I].isGenericType())1134      continue;1135    // Generic instructions specify type equality constraints between some of1136    // their operands. Make sure these are consistent.1137    size_t TypeIdx = MCID.operands()[I].getGenericTypeIndex();1138    Types.resize(std::max(TypeIdx + 1, Types.size()));1139 1140    const MachineOperand *MO = &MI->getOperand(I);1141    if (!MO->isReg()) {1142      report("generic instruction must use register operands", MI);1143      continue;1144    }1145 1146    LLT OpTy = MRI->getType(MO->getReg());1147    // Don't report a type mismatch if there is no actual mismatch, only a1148    // type missing, to reduce noise:1149    if (OpTy.isValid()) {1150      // Only the first valid type for a type index will be printed: don't1151      // overwrite it later so it's always clear which type was expected:1152      if (!Types[TypeIdx].isValid())1153        Types[TypeIdx] = OpTy;1154      else if (Types[TypeIdx] != OpTy)1155        report("Type mismatch in generic instruction", MO, I, OpTy);1156    } else {1157      // Generic instructions must have types attached to their operands.1158      report("Generic instruction is missing a virtual register type", MO, I);1159    }1160  }1161 1162  // Generic opcodes must not have physical register operands.1163  for (unsigned I = 0; I < MI->getNumOperands(); ++I) {1164    const MachineOperand *MO = &MI->getOperand(I);1165    if (MO->isReg() && MO->getReg().isPhysical())1166      report("Generic instruction cannot have physical register", MO, I);1167  }1168 1169  // Avoid out of bounds in checks below. This was already reported earlier.1170  if (MI->getNumOperands() < MCID.getNumOperands())1171    return;1172 1173  StringRef ErrorInfo;1174  if (!TII->verifyInstruction(*MI, ErrorInfo))1175    report(ErrorInfo.data(), MI);1176 1177  // Verify properties of various specific instruction types1178  unsigned Opc = MI->getOpcode();1179  switch (Opc) {1180  case TargetOpcode::G_ASSERT_SEXT:1181  case TargetOpcode::G_ASSERT_ZEXT: {1182    std::string OpcName =1183        Opc == TargetOpcode::G_ASSERT_ZEXT ? "G_ASSERT_ZEXT" : "G_ASSERT_SEXT";1184    if (!MI->getOperand(2).isImm()) {1185      report(Twine(OpcName, " expects an immediate operand #2"), MI);1186      break;1187    }1188 1189    Register Dst = MI->getOperand(0).getReg();1190    Register Src = MI->getOperand(1).getReg();1191    LLT SrcTy = MRI->getType(Src);1192    int64_t Imm = MI->getOperand(2).getImm();1193    if (Imm <= 0) {1194      report(Twine(OpcName, " size must be >= 1"), MI);1195      break;1196    }1197 1198    if (Imm >= SrcTy.getScalarSizeInBits()) {1199      report(Twine(OpcName, " size must be less than source bit width"), MI);1200      break;1201    }1202 1203    const RegisterBank *SrcRB = RBI->getRegBank(Src, *MRI, *TRI);1204    const RegisterBank *DstRB = RBI->getRegBank(Dst, *MRI, *TRI);1205 1206    // Allow only the source bank to be set.1207    if ((SrcRB && DstRB && SrcRB != DstRB) || (DstRB && !SrcRB)) {1208      report(Twine(OpcName, " cannot change register bank"), MI);1209      break;1210    }1211 1212    // Don't allow a class change. Do allow member class->regbank.1213    const TargetRegisterClass *DstRC = MRI->getRegClassOrNull(Dst);1214    if (DstRC && DstRC != MRI->getRegClassOrNull(Src)) {1215      report(1216          Twine(OpcName, " source and destination register classes must match"),1217          MI);1218      break;1219    }1220 1221    break;1222  }1223 1224  case TargetOpcode::G_CONSTANT:1225  case TargetOpcode::G_FCONSTANT: {1226    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1227    if (DstTy.isVector())1228      report("Instruction cannot use a vector result type", MI);1229 1230    if (MI->getOpcode() == TargetOpcode::G_CONSTANT) {1231      if (!MI->getOperand(1).isCImm()) {1232        report("G_CONSTANT operand must be cimm", MI);1233        break;1234      }1235 1236      const ConstantInt *CI = MI->getOperand(1).getCImm();1237      if (CI->getBitWidth() != DstTy.getSizeInBits())1238        report("inconsistent constant size", MI);1239    } else {1240      if (!MI->getOperand(1).isFPImm()) {1241        report("G_FCONSTANT operand must be fpimm", MI);1242        break;1243      }1244      const ConstantFP *CF = MI->getOperand(1).getFPImm();1245 1246      if (APFloat::getSizeInBits(CF->getValueAPF().getSemantics()) !=1247          DstTy.getSizeInBits()) {1248        report("inconsistent constant size", MI);1249      }1250    }1251 1252    break;1253  }1254  case TargetOpcode::G_LOAD:1255  case TargetOpcode::G_STORE:1256  case TargetOpcode::G_ZEXTLOAD:1257  case TargetOpcode::G_SEXTLOAD: {1258    LLT ValTy = MRI->getType(MI->getOperand(0).getReg());1259    LLT PtrTy = MRI->getType(MI->getOperand(1).getReg());1260    if (!PtrTy.isPointer())1261      report("Generic memory instruction must access a pointer", MI);1262 1263    // Generic loads and stores must have a single MachineMemOperand1264    // describing that access.1265    if (!MI->hasOneMemOperand()) {1266      report("Generic instruction accessing memory must have one mem operand",1267             MI);1268    } else {1269      const MachineMemOperand &MMO = **MI->memoperands_begin();1270      if (MI->getOpcode() == TargetOpcode::G_ZEXTLOAD ||1271          MI->getOpcode() == TargetOpcode::G_SEXTLOAD) {1272        if (TypeSize::isKnownGE(MMO.getSizeInBits().getValue(),1273                                ValTy.getSizeInBits()))1274          report("Generic extload must have a narrower memory type", MI);1275      } else if (MI->getOpcode() == TargetOpcode::G_LOAD) {1276        if (TypeSize::isKnownGT(MMO.getSize().getValue(),1277                                ValTy.getSizeInBytes()))1278          report("load memory size cannot exceed result size", MI);1279 1280        if (MMO.getRanges()) {1281          ConstantInt *i =1282              mdconst::extract<ConstantInt>(MMO.getRanges()->getOperand(0));1283          const LLT RangeTy = LLT::scalar(i->getIntegerType()->getBitWidth());1284          const LLT MemTy = MMO.getMemoryType();1285          if (MemTy.getScalarType() != RangeTy ||1286              ValTy.isScalar() != MemTy.isScalar() ||1287              (ValTy.isVector() &&1288               ValTy.getNumElements() != MemTy.getNumElements())) {1289            report("range is incompatible with the result type", MI);1290          }1291        }1292      } else if (MI->getOpcode() == TargetOpcode::G_STORE) {1293        if (TypeSize::isKnownLT(ValTy.getSizeInBytes(),1294                                MMO.getSize().getValue()))1295          report("store memory size cannot exceed value size", MI);1296      }1297 1298      const AtomicOrdering Order = MMO.getSuccessOrdering();1299      if (Opc == TargetOpcode::G_STORE) {1300        if (Order == AtomicOrdering::Acquire ||1301            Order == AtomicOrdering::AcquireRelease)1302          report("atomic store cannot use acquire ordering", MI);1303 1304      } else {1305        if (Order == AtomicOrdering::Release ||1306            Order == AtomicOrdering::AcquireRelease)1307          report("atomic load cannot use release ordering", MI);1308      }1309    }1310 1311    break;1312  }1313  case TargetOpcode::G_PHI: {1314    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1315    if (!DstTy.isValid() || !all_of(drop_begin(MI->operands()),1316                                    [this, &DstTy](const MachineOperand &MO) {1317                                      if (!MO.isReg())1318                                        return true;1319                                      LLT Ty = MRI->getType(MO.getReg());1320                                      if (!Ty.isValid() || (Ty != DstTy))1321                                        return false;1322                                      return true;1323                                    }))1324      report("Generic Instruction G_PHI has operands with incompatible/missing "1325             "types",1326             MI);1327    break;1328  }1329  case TargetOpcode::G_BITCAST: {1330    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1331    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());1332    if (!DstTy.isValid() || !SrcTy.isValid())1333      break;1334 1335    if (SrcTy.isPointer() != DstTy.isPointer())1336      report("bitcast cannot convert between pointers and other types", MI);1337 1338    if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())1339      report("bitcast sizes must match", MI);1340 1341    if (SrcTy == DstTy)1342      report("bitcast must change the type", MI);1343 1344    break;1345  }1346  case TargetOpcode::G_INTTOPTR:1347  case TargetOpcode::G_PTRTOINT:1348  case TargetOpcode::G_ADDRSPACE_CAST: {1349    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1350    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());1351    if (!DstTy.isValid() || !SrcTy.isValid())1352      break;1353 1354    verifyVectorElementMatch(DstTy, SrcTy, MI);1355 1356    DstTy = DstTy.getScalarType();1357    SrcTy = SrcTy.getScalarType();1358 1359    if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) {1360      if (!DstTy.isPointer())1361        report("inttoptr result type must be a pointer", MI);1362      if (SrcTy.isPointer())1363        report("inttoptr source type must not be a pointer", MI);1364    } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) {1365      if (!SrcTy.isPointer())1366        report("ptrtoint source type must be a pointer", MI);1367      if (DstTy.isPointer())1368        report("ptrtoint result type must not be a pointer", MI);1369    } else {1370      assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);1371      if (!SrcTy.isPointer() || !DstTy.isPointer())1372        report("addrspacecast types must be pointers", MI);1373      else {1374        if (SrcTy.getAddressSpace() == DstTy.getAddressSpace())1375          report("addrspacecast must convert different address spaces", MI);1376      }1377    }1378 1379    break;1380  }1381  case TargetOpcode::G_PTR_ADD: {1382    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1383    LLT PtrTy = MRI->getType(MI->getOperand(1).getReg());1384    LLT OffsetTy = MRI->getType(MI->getOperand(2).getReg());1385    if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid())1386      break;1387 1388    if (!PtrTy.isPointerOrPointerVector())1389      report("gep first operand must be a pointer", MI);1390 1391    if (OffsetTy.isPointerOrPointerVector())1392      report("gep offset operand must not be a pointer", MI);1393 1394    if (PtrTy.isPointerOrPointerVector()) {1395      const DataLayout &DL = MF->getDataLayout();1396      unsigned AS = PtrTy.getAddressSpace();1397      unsigned IndexSizeInBits = DL.getIndexSize(AS) * 8;1398      if (OffsetTy.getScalarSizeInBits() != IndexSizeInBits) {1399        report("gep offset operand must match index size for address space",1400               MI);1401      }1402    }1403 1404    // TODO: Is the offset allowed to be a scalar with a vector?1405    break;1406  }1407  case TargetOpcode::G_PTRMASK: {1408    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1409    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());1410    LLT MaskTy = MRI->getType(MI->getOperand(2).getReg());1411    if (!DstTy.isValid() || !SrcTy.isValid() || !MaskTy.isValid())1412      break;1413 1414    if (!DstTy.isPointerOrPointerVector())1415      report("ptrmask result type must be a pointer", MI);1416 1417    if (!MaskTy.getScalarType().isScalar())1418      report("ptrmask mask type must be an integer", MI);1419 1420    verifyVectorElementMatch(DstTy, MaskTy, MI);1421    break;1422  }1423  case TargetOpcode::G_SEXT:1424  case TargetOpcode::G_ZEXT:1425  case TargetOpcode::G_ANYEXT:1426  case TargetOpcode::G_TRUNC:1427  case TargetOpcode::G_TRUNC_SSAT_S:1428  case TargetOpcode::G_TRUNC_SSAT_U:1429  case TargetOpcode::G_TRUNC_USAT_U:1430  case TargetOpcode::G_FPEXT:1431  case TargetOpcode::G_FPTRUNC: {1432    // Number of operands and presense of types is already checked (and1433    // reported in case of any issues), so no need to report them again. As1434    // we're trying to report as many issues as possible at once, however, the1435    // instructions aren't guaranteed to have the right number of operands or1436    // types attached to them at this point1437    assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}");1438    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1439    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());1440    if (!DstTy.isValid() || !SrcTy.isValid())1441      break;1442 1443    if (DstTy.isPointerOrPointerVector() || SrcTy.isPointerOrPointerVector())1444      report("Generic extend/truncate can not operate on pointers", MI);1445 1446    verifyVectorElementMatch(DstTy, SrcTy, MI);1447 1448    unsigned DstSize = DstTy.getScalarSizeInBits();1449    unsigned SrcSize = SrcTy.getScalarSizeInBits();1450    switch (MI->getOpcode()) {1451    default:1452      if (DstSize <= SrcSize)1453        report("Generic extend has destination type no larger than source", MI);1454      break;1455    case TargetOpcode::G_TRUNC:1456    case TargetOpcode::G_TRUNC_SSAT_S:1457    case TargetOpcode::G_TRUNC_SSAT_U:1458    case TargetOpcode::G_TRUNC_USAT_U:1459    case TargetOpcode::G_FPTRUNC:1460      if (DstSize >= SrcSize)1461        report("Generic truncate has destination type no smaller than source",1462               MI);1463      break;1464    }1465    break;1466  }1467  case TargetOpcode::G_SELECT: {1468    LLT SelTy = MRI->getType(MI->getOperand(0).getReg());1469    LLT CondTy = MRI->getType(MI->getOperand(1).getReg());1470    if (!SelTy.isValid() || !CondTy.isValid())1471      break;1472 1473    // Scalar condition select on a vector is valid.1474    if (CondTy.isVector())1475      verifyVectorElementMatch(SelTy, CondTy, MI);1476    break;1477  }1478  case TargetOpcode::G_MERGE_VALUES: {1479    // G_MERGE_VALUES should only be used to merge scalars into a larger scalar,1480    // e.g. s2N = MERGE sN, sN1481    // Merging multiple scalars into a vector is not allowed, should use1482    // G_BUILD_VECTOR for that.1483    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1484    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());1485    if (DstTy.isVector() || SrcTy.isVector())1486      report("G_MERGE_VALUES cannot operate on vectors", MI);1487 1488    const unsigned NumOps = MI->getNumOperands();1489    if (DstTy.getSizeInBits() != SrcTy.getSizeInBits() * (NumOps - 1))1490      report("G_MERGE_VALUES result size is inconsistent", MI);1491 1492    for (unsigned I = 2; I != NumOps; ++I) {1493      if (MRI->getType(MI->getOperand(I).getReg()) != SrcTy)1494        report("G_MERGE_VALUES source types do not match", MI);1495    }1496 1497    break;1498  }1499  case TargetOpcode::G_UNMERGE_VALUES: {1500    unsigned NumDsts = MI->getNumOperands() - 1;1501    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1502    for (unsigned i = 1; i < NumDsts; ++i) {1503      if (MRI->getType(MI->getOperand(i).getReg()) != DstTy) {1504        report("G_UNMERGE_VALUES destination types do not match", MI);1505        break;1506      }1507    }1508 1509    LLT SrcTy = MRI->getType(MI->getOperand(NumDsts).getReg());1510    if (DstTy.isVector()) {1511      // This case is the converse of G_CONCAT_VECTORS.1512      if (!SrcTy.isVector() ||1513          (SrcTy.getScalarType() != DstTy.getScalarType() &&1514           !SrcTy.isPointerVector()) ||1515          SrcTy.isScalableVector() != DstTy.isScalableVector() ||1516          SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())1517        report("G_UNMERGE_VALUES source operand does not match vector "1518               "destination operands",1519               MI);1520    } else if (SrcTy.isVector()) {1521      // This case is the converse of G_BUILD_VECTOR, but relaxed to allow1522      // mismatched types as long as the total size matches:1523      //   %0:_(s64), %1:_(s64) = G_UNMERGE_VALUES %2:_(<4 x s32>)1524      if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits())1525        report("G_UNMERGE_VALUES vector source operand does not match scalar "1526               "destination operands",1527               MI);1528    } else {1529      // This case is the converse of G_MERGE_VALUES.1530      if (SrcTy.getSizeInBits() != NumDsts * DstTy.getSizeInBits()) {1531        report("G_UNMERGE_VALUES scalar source operand does not match scalar "1532               "destination operands",1533               MI);1534      }1535    }1536    break;1537  }1538  case TargetOpcode::G_BUILD_VECTOR: {1539    // Source types must be scalars, dest type a vector. Total size of scalars1540    // must match the dest vector size.1541    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1542    LLT SrcEltTy = MRI->getType(MI->getOperand(1).getReg());1543    if (!DstTy.isVector() || SrcEltTy.isVector()) {1544      report("G_BUILD_VECTOR must produce a vector from scalar operands", MI);1545      break;1546    }1547 1548    if (DstTy.getElementType() != SrcEltTy)1549      report("G_BUILD_VECTOR result element type must match source type", MI);1550 1551    if (DstTy.getNumElements() != MI->getNumOperands() - 1)1552      report("G_BUILD_VECTOR must have an operand for each element", MI);1553 1554    for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 2))1555      if (MRI->getType(MI->getOperand(1).getReg()) != MRI->getType(MO.getReg()))1556        report("G_BUILD_VECTOR source operand types are not homogeneous", MI);1557 1558    break;1559  }1560  case TargetOpcode::G_BUILD_VECTOR_TRUNC: {1561    // Source types must be scalars, dest type a vector. Scalar types must be1562    // larger than the dest vector elt type, as this is a truncating operation.1563    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1564    LLT SrcEltTy = MRI->getType(MI->getOperand(1).getReg());1565    if (!DstTy.isVector() || SrcEltTy.isVector())1566      report("G_BUILD_VECTOR_TRUNC must produce a vector from scalar operands",1567             MI);1568    for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 2))1569      if (MRI->getType(MI->getOperand(1).getReg()) != MRI->getType(MO.getReg()))1570        report("G_BUILD_VECTOR_TRUNC source operand types are not homogeneous",1571               MI);1572    if (SrcEltTy.getSizeInBits() <= DstTy.getElementType().getSizeInBits())1573      report("G_BUILD_VECTOR_TRUNC source operand types are not larger than "1574             "dest elt type",1575             MI);1576    break;1577  }1578  case TargetOpcode::G_CONCAT_VECTORS: {1579    // Source types should be vectors, and total size should match the dest1580    // vector size.1581    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1582    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());1583    if (!DstTy.isVector() || !SrcTy.isVector())1584      report("G_CONCAT_VECTOR requires vector source and destination operands",1585             MI);1586 1587    if (MI->getNumOperands() < 3)1588      report("G_CONCAT_VECTOR requires at least 2 source operands", MI);1589 1590    for (const MachineOperand &MO : llvm::drop_begin(MI->operands(), 2))1591      if (MRI->getType(MI->getOperand(1).getReg()) != MRI->getType(MO.getReg()))1592        report("G_CONCAT_VECTOR source operand types are not homogeneous", MI);1593    if (DstTy.getElementCount() !=1594        SrcTy.getElementCount() * (MI->getNumOperands() - 1))1595      report("G_CONCAT_VECTOR num dest and source elements should match", MI);1596    break;1597  }1598  case TargetOpcode::G_ICMP:1599  case TargetOpcode::G_FCMP: {1600    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1601    LLT SrcTy = MRI->getType(MI->getOperand(2).getReg());1602 1603    if ((DstTy.isVector() != SrcTy.isVector()) ||1604        (DstTy.isVector() &&1605         DstTy.getElementCount() != SrcTy.getElementCount()))1606      report("Generic vector icmp/fcmp must preserve number of lanes", MI);1607 1608    break;1609  }1610  case TargetOpcode::G_SCMP:1611  case TargetOpcode::G_UCMP: {1612    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1613    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());1614 1615    if (SrcTy.isPointerOrPointerVector()) {1616      report("Generic scmp/ucmp does not support pointers as operands", MI);1617      break;1618    }1619 1620    if (DstTy.isPointerOrPointerVector()) {1621      report("Generic scmp/ucmp does not support pointers as a result", MI);1622      break;1623    }1624 1625    if (DstTy.getScalarSizeInBits() < 2) {1626      report("Result type must be at least 2 bits wide", MI);1627      break;1628    }1629 1630    if ((DstTy.isVector() != SrcTy.isVector()) ||1631        (DstTy.isVector() &&1632         DstTy.getElementCount() != SrcTy.getElementCount())) {1633      report("Generic vector scmp/ucmp must preserve number of lanes", MI);1634      break;1635    }1636 1637    break;1638  }1639  case TargetOpcode::G_EXTRACT: {1640    const MachineOperand &SrcOp = MI->getOperand(1);1641    if (!SrcOp.isReg()) {1642      report("extract source must be a register", MI);1643      break;1644    }1645 1646    const MachineOperand &OffsetOp = MI->getOperand(2);1647    if (!OffsetOp.isImm()) {1648      report("extract offset must be a constant", MI);1649      break;1650    }1651 1652    unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits();1653    unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits();1654    if (SrcSize == DstSize)1655      report("extract source must be larger than result", MI);1656 1657    if (DstSize + OffsetOp.getImm() > SrcSize)1658      report("extract reads past end of register", MI);1659    break;1660  }1661  case TargetOpcode::G_INSERT: {1662    const MachineOperand &SrcOp = MI->getOperand(2);1663    if (!SrcOp.isReg()) {1664      report("insert source must be a register", MI);1665      break;1666    }1667 1668    const MachineOperand &OffsetOp = MI->getOperand(3);1669    if (!OffsetOp.isImm()) {1670      report("insert offset must be a constant", MI);1671      break;1672    }1673 1674    unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits();1675    unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits();1676 1677    if (DstSize <= SrcSize)1678      report("inserted size must be smaller than total register", MI);1679 1680    if (SrcSize + OffsetOp.getImm() > DstSize)1681      report("insert writes past end of register", MI);1682 1683    break;1684  }1685  case TargetOpcode::G_JUMP_TABLE: {1686    if (!MI->getOperand(1).isJTI())1687      report("G_JUMP_TABLE source operand must be a jump table index", MI);1688    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1689    if (!DstTy.isPointer())1690      report("G_JUMP_TABLE dest operand must have a pointer type", MI);1691    break;1692  }1693  case TargetOpcode::G_BRJT: {1694    if (!MRI->getType(MI->getOperand(0).getReg()).isPointer())1695      report("G_BRJT src operand 0 must be a pointer type", MI);1696 1697    if (!MI->getOperand(1).isJTI())1698      report("G_BRJT src operand 1 must be a jump table index", MI);1699 1700    const auto &IdxOp = MI->getOperand(2);1701    if (!IdxOp.isReg() || MRI->getType(IdxOp.getReg()).isPointer())1702      report("G_BRJT src operand 2 must be a scalar reg type", MI);1703    break;1704  }1705  case TargetOpcode::G_INTRINSIC:1706  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:1707  case TargetOpcode::G_INTRINSIC_CONVERGENT:1708  case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: {1709    // TODO: Should verify number of def and use operands, but the current1710    // interface requires passing in IR types for mangling.1711    const MachineOperand &IntrIDOp = MI->getOperand(MI->getNumExplicitDefs());1712    if (!IntrIDOp.isIntrinsicID()) {1713      report("G_INTRINSIC first src operand must be an intrinsic ID", MI);1714      break;1715    }1716 1717    if (!verifyGIntrinsicSideEffects(MI))1718      break;1719    if (!verifyGIntrinsicConvergence(MI))1720      break;1721 1722    break;1723  }1724  case TargetOpcode::G_SEXT_INREG: {1725    if (!MI->getOperand(2).isImm()) {1726      report("G_SEXT_INREG expects an immediate operand #2", MI);1727      break;1728    }1729 1730    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());1731    int64_t Imm = MI->getOperand(2).getImm();1732    if (Imm <= 0)1733      report("G_SEXT_INREG size must be >= 1", MI);1734    if (Imm >= SrcTy.getScalarSizeInBits())1735      report("G_SEXT_INREG size must be less than source bit width", MI);1736    break;1737  }1738  case TargetOpcode::G_BSWAP: {1739    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1740    if (DstTy.getScalarSizeInBits() % 16 != 0)1741      report("G_BSWAP size must be a multiple of 16 bits", MI);1742    break;1743  }1744  case TargetOpcode::G_VSCALE: {1745    if (!MI->getOperand(1).isCImm()) {1746      report("G_VSCALE operand must be cimm", MI);1747      break;1748    }1749    if (MI->getOperand(1).getCImm()->isZero()) {1750      report("G_VSCALE immediate cannot be zero", MI);1751      break;1752    }1753    break;1754  }1755  case TargetOpcode::G_STEP_VECTOR: {1756    if (!MI->getOperand(1).isCImm()) {1757      report("operand must be cimm", MI);1758      break;1759    }1760 1761    if (!MI->getOperand(1).getCImm()->getValue().isStrictlyPositive()) {1762      report("step must be > 0", MI);1763      break;1764    }1765 1766    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1767    if (!DstTy.isScalableVector()) {1768      report("Destination type must be a scalable vector", MI);1769      break;1770    }1771 1772    // <vscale x 2 x p0>1773    if (!DstTy.getElementType().isScalar()) {1774      report("Destination element type must be scalar", MI);1775      break;1776    }1777 1778    if (MI->getOperand(1).getCImm()->getBitWidth() !=1779        DstTy.getElementType().getScalarSizeInBits()) {1780      report("step bitwidth differs from result type element bitwidth", MI);1781      break;1782    }1783    break;1784  }1785  case TargetOpcode::G_INSERT_SUBVECTOR: {1786    const MachineOperand &Src0Op = MI->getOperand(1);1787    if (!Src0Op.isReg()) {1788      report("G_INSERT_SUBVECTOR first source must be a register", MI);1789      break;1790    }1791 1792    const MachineOperand &Src1Op = MI->getOperand(2);1793    if (!Src1Op.isReg()) {1794      report("G_INSERT_SUBVECTOR second source must be a register", MI);1795      break;1796    }1797 1798    const MachineOperand &IndexOp = MI->getOperand(3);1799    if (!IndexOp.isImm()) {1800      report("G_INSERT_SUBVECTOR index must be an immediate", MI);1801      break;1802    }1803 1804    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1805    LLT Src1Ty = MRI->getType(Src1Op.getReg());1806 1807    if (!DstTy.isVector()) {1808      report("Destination type must be a vector", MI);1809      break;1810    }1811 1812    if (!Src1Ty.isVector()) {1813      report("Second source must be a vector", MI);1814      break;1815    }1816 1817    if (DstTy.getElementType() != Src1Ty.getElementType()) {1818      report("Element type of vectors must be the same", MI);1819      break;1820    }1821 1822    if (Src1Ty.isScalable() != DstTy.isScalable()) {1823      report("Vector types must both be fixed or both be scalable", MI);1824      break;1825    }1826 1827    if (ElementCount::isKnownGT(Src1Ty.getElementCount(),1828                                DstTy.getElementCount())) {1829      report("Second source must be smaller than destination vector", MI);1830      break;1831    }1832 1833    uint64_t Idx = IndexOp.getImm();1834    uint64_t Src1MinLen = Src1Ty.getElementCount().getKnownMinValue();1835    if (IndexOp.getImm() % Src1MinLen != 0) {1836      report("Index must be a multiple of the second source vector's "1837             "minimum vector length",1838             MI);1839      break;1840    }1841 1842    uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();1843    if (Idx >= DstMinLen || Idx + Src1MinLen > DstMinLen) {1844      report("Subvector type and index must not cause insert to overrun the "1845             "vector being inserted into",1846             MI);1847      break;1848    }1849 1850    break;1851  }1852  case TargetOpcode::G_EXTRACT_SUBVECTOR: {1853    const MachineOperand &SrcOp = MI->getOperand(1);1854    if (!SrcOp.isReg()) {1855      report("G_EXTRACT_SUBVECTOR first source must be a register", MI);1856      break;1857    }1858 1859    const MachineOperand &IndexOp = MI->getOperand(2);1860    if (!IndexOp.isImm()) {1861      report("G_EXTRACT_SUBVECTOR index must be an immediate", MI);1862      break;1863    }1864 1865    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1866    LLT SrcTy = MRI->getType(SrcOp.getReg());1867 1868    if (!DstTy.isVector()) {1869      report("Destination type must be a vector", MI);1870      break;1871    }1872 1873    if (!SrcTy.isVector()) {1874      report("Source must be a vector", MI);1875      break;1876    }1877 1878    if (DstTy.getElementType() != SrcTy.getElementType()) {1879      report("Element type of vectors must be the same", MI);1880      break;1881    }1882 1883    if (SrcTy.isScalable() != DstTy.isScalable()) {1884      report("Vector types must both be fixed or both be scalable", MI);1885      break;1886    }1887 1888    if (ElementCount::isKnownGT(DstTy.getElementCount(),1889                                SrcTy.getElementCount())) {1890      report("Destination vector must be smaller than source vector", MI);1891      break;1892    }1893 1894    uint64_t Idx = IndexOp.getImm();1895    uint64_t DstMinLen = DstTy.getElementCount().getKnownMinValue();1896    if (Idx % DstMinLen != 0) {1897      report("Index must be a multiple of the destination vector's minimum "1898             "vector length",1899             MI);1900      break;1901    }1902 1903    uint64_t SrcMinLen = SrcTy.getElementCount().getKnownMinValue();1904    if (Idx >= SrcMinLen || Idx + DstMinLen > SrcMinLen) {1905      report("Destination type and index must not cause extract to overrun the "1906             "source vector",1907             MI);1908      break;1909    }1910 1911    break;1912  }1913  case TargetOpcode::G_SHUFFLE_VECTOR: {1914    const MachineOperand &MaskOp = MI->getOperand(3);1915    if (!MaskOp.isShuffleMask()) {1916      report("Incorrect mask operand type for G_SHUFFLE_VECTOR", MI);1917      break;1918    }1919 1920    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1921    LLT Src0Ty = MRI->getType(MI->getOperand(1).getReg());1922    LLT Src1Ty = MRI->getType(MI->getOperand(2).getReg());1923 1924    if (Src0Ty != Src1Ty)1925      report("Source operands must be the same type", MI);1926 1927    if (Src0Ty.getScalarType() != DstTy.getScalarType()) {1928      report("G_SHUFFLE_VECTOR cannot change element type", MI);1929      break;1930    }1931    if (!Src0Ty.isVector()) {1932      report("G_SHUFFLE_VECTOR must have vector src", MI);1933      break;1934    }1935    if (!DstTy.isVector()) {1936      report("G_SHUFFLE_VECTOR must have vector dst", MI);1937      break;1938    }1939 1940    // Don't check that all operands are vector because scalars are used in1941    // place of 1 element vectors.1942    int SrcNumElts = Src0Ty.getNumElements();1943    int DstNumElts = DstTy.getNumElements();1944 1945    ArrayRef<int> MaskIdxes = MaskOp.getShuffleMask();1946 1947    if (static_cast<int>(MaskIdxes.size()) != DstNumElts)1948      report("Wrong result type for shufflemask", MI);1949 1950    for (int Idx : MaskIdxes) {1951      if (Idx < 0)1952        continue;1953 1954      if (Idx >= 2 * SrcNumElts)1955        report("Out of bounds shuffle index", MI);1956    }1957 1958    break;1959  }1960 1961  case TargetOpcode::G_SPLAT_VECTOR: {1962    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1963    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());1964 1965    if (!DstTy.isScalableVector()) {1966      report("Destination type must be a scalable vector", MI);1967      break;1968    }1969 1970    if (!SrcTy.isScalar() && !SrcTy.isPointer()) {1971      report("Source type must be a scalar or pointer", MI);1972      break;1973    }1974 1975    if (TypeSize::isKnownGT(DstTy.getElementType().getSizeInBits(),1976                            SrcTy.getSizeInBits())) {1977      report("Element type of the destination must be the same size or smaller "1978             "than the source type",1979             MI);1980      break;1981    }1982 1983    break;1984  }1985  case TargetOpcode::G_EXTRACT_VECTOR_ELT: {1986    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());1987    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());1988    LLT IdxTy = MRI->getType(MI->getOperand(2).getReg());1989 1990    if (!DstTy.isScalar() && !DstTy.isPointer()) {1991      report("Destination type must be a scalar or pointer", MI);1992      break;1993    }1994 1995    if (!SrcTy.isVector()) {1996      report("First source must be a vector", MI);1997      break;1998    }1999 2000    auto TLI = MF->getSubtarget().getTargetLowering();2001    if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(MF->getDataLayout())) {2002      report("Index type must match VectorIdxTy", MI);2003      break;2004    }2005 2006    break;2007  }2008  case TargetOpcode::G_INSERT_VECTOR_ELT: {2009    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());2010    LLT VecTy = MRI->getType(MI->getOperand(1).getReg());2011    LLT ScaTy = MRI->getType(MI->getOperand(2).getReg());2012    LLT IdxTy = MRI->getType(MI->getOperand(3).getReg());2013 2014    if (!DstTy.isVector()) {2015      report("Destination type must be a vector", MI);2016      break;2017    }2018 2019    if (VecTy != DstTy) {2020      report("Destination type and vector type must match", MI);2021      break;2022    }2023 2024    if (!ScaTy.isScalar() && !ScaTy.isPointer()) {2025      report("Inserted element must be a scalar or pointer", MI);2026      break;2027    }2028 2029    auto TLI = MF->getSubtarget().getTargetLowering();2030    if (IdxTy.getSizeInBits() != TLI->getVectorIdxWidth(MF->getDataLayout())) {2031      report("Index type must match VectorIdxTy", MI);2032      break;2033    }2034 2035    break;2036  }2037  case TargetOpcode::G_DYN_STACKALLOC: {2038    const MachineOperand &DstOp = MI->getOperand(0);2039    const MachineOperand &AllocOp = MI->getOperand(1);2040    const MachineOperand &AlignOp = MI->getOperand(2);2041 2042    if (!DstOp.isReg() || !MRI->getType(DstOp.getReg()).isPointer()) {2043      report("dst operand 0 must be a pointer type", MI);2044      break;2045    }2046 2047    if (!AllocOp.isReg() || !MRI->getType(AllocOp.getReg()).isScalar()) {2048      report("src operand 1 must be a scalar reg type", MI);2049      break;2050    }2051 2052    if (!AlignOp.isImm()) {2053      report("src operand 2 must be an immediate type", MI);2054      break;2055    }2056    break;2057  }2058  case TargetOpcode::G_MEMCPY_INLINE:2059  case TargetOpcode::G_MEMCPY:2060  case TargetOpcode::G_MEMMOVE: {2061    ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();2062    if (MMOs.size() != 2) {2063      report("memcpy/memmove must have 2 memory operands", MI);2064      break;2065    }2066 2067    if ((!MMOs[0]->isStore() || MMOs[0]->isLoad()) ||2068        (MMOs[1]->isStore() || !MMOs[1]->isLoad())) {2069      report("wrong memory operand types", MI);2070      break;2071    }2072 2073    if (MMOs[0]->getSize() != MMOs[1]->getSize())2074      report("inconsistent memory operand sizes", MI);2075 2076    LLT DstPtrTy = MRI->getType(MI->getOperand(0).getReg());2077    LLT SrcPtrTy = MRI->getType(MI->getOperand(1).getReg());2078 2079    if (!DstPtrTy.isPointer() || !SrcPtrTy.isPointer()) {2080      report("memory instruction operand must be a pointer", MI);2081      break;2082    }2083 2084    if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())2085      report("inconsistent store address space", MI);2086    if (SrcPtrTy.getAddressSpace() != MMOs[1]->getAddrSpace())2087      report("inconsistent load address space", MI);2088 2089    if (Opc != TargetOpcode::G_MEMCPY_INLINE)2090      if (!MI->getOperand(3).isImm() || (MI->getOperand(3).getImm() & ~1LL))2091        report("'tail' flag (operand 3) must be an immediate 0 or 1", MI);2092 2093    break;2094  }2095  case TargetOpcode::G_BZERO:2096  case TargetOpcode::G_MEMSET: {2097    ArrayRef<MachineMemOperand *> MMOs = MI->memoperands();2098    std::string Name = Opc == TargetOpcode::G_MEMSET ? "memset" : "bzero";2099    if (MMOs.size() != 1) {2100      report(Twine(Name, " must have 1 memory operand"), MI);2101      break;2102    }2103 2104    if ((!MMOs[0]->isStore() || MMOs[0]->isLoad())) {2105      report(Twine(Name, " memory operand must be a store"), MI);2106      break;2107    }2108 2109    LLT DstPtrTy = MRI->getType(MI->getOperand(0).getReg());2110    if (!DstPtrTy.isPointer()) {2111      report(Twine(Name, " operand must be a pointer"), MI);2112      break;2113    }2114 2115    if (DstPtrTy.getAddressSpace() != MMOs[0]->getAddrSpace())2116      report("inconsistent " + Twine(Name, " address space"), MI);2117 2118    if (!MI->getOperand(MI->getNumOperands() - 1).isImm() ||2119        (MI->getOperand(MI->getNumOperands() - 1).getImm() & ~1LL))2120      report("'tail' flag (last operand) must be an immediate 0 or 1", MI);2121 2122    break;2123  }2124  case TargetOpcode::G_UBSANTRAP: {2125    const MachineOperand &KindOp = MI->getOperand(0);2126    if (!MI->getOperand(0).isImm()) {2127      report("Crash kind must be an immediate", &KindOp, 0);2128      break;2129    }2130    int64_t Kind = MI->getOperand(0).getImm();2131    if (!isInt<8>(Kind))2132      report("Crash kind must be 8 bit wide", &KindOp, 0);2133    break;2134  }2135  case TargetOpcode::G_VECREDUCE_SEQ_FADD:2136  case TargetOpcode::G_VECREDUCE_SEQ_FMUL: {2137    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());2138    LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg());2139    LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg());2140    if (!DstTy.isScalar())2141      report("Vector reduction requires a scalar destination type", MI);2142    if (!Src1Ty.isScalar())2143      report("Sequential FADD/FMUL vector reduction requires a scalar 1st operand", MI);2144    if (!Src2Ty.isVector())2145      report("Sequential FADD/FMUL vector reduction must have a vector 2nd operand", MI);2146    break;2147  }2148  case TargetOpcode::G_VECREDUCE_FADD:2149  case TargetOpcode::G_VECREDUCE_FMUL:2150  case TargetOpcode::G_VECREDUCE_FMAX:2151  case TargetOpcode::G_VECREDUCE_FMIN:2152  case TargetOpcode::G_VECREDUCE_FMAXIMUM:2153  case TargetOpcode::G_VECREDUCE_FMINIMUM:2154  case TargetOpcode::G_VECREDUCE_ADD:2155  case TargetOpcode::G_VECREDUCE_MUL:2156  case TargetOpcode::G_VECREDUCE_AND:2157  case TargetOpcode::G_VECREDUCE_OR:2158  case TargetOpcode::G_VECREDUCE_XOR:2159  case TargetOpcode::G_VECREDUCE_SMAX:2160  case TargetOpcode::G_VECREDUCE_SMIN:2161  case TargetOpcode::G_VECREDUCE_UMAX:2162  case TargetOpcode::G_VECREDUCE_UMIN: {2163    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());2164    if (!DstTy.isScalar())2165      report("Vector reduction requires a scalar destination type", MI);2166    break;2167  }2168 2169  case TargetOpcode::G_SBFX:2170  case TargetOpcode::G_UBFX: {2171    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());2172    if (DstTy.isVector()) {2173      report("Bitfield extraction is not supported on vectors", MI);2174      break;2175    }2176    break;2177  }2178  case TargetOpcode::G_SHL:2179  case TargetOpcode::G_LSHR:2180  case TargetOpcode::G_ASHR:2181  case TargetOpcode::G_ROTR:2182  case TargetOpcode::G_ROTL: {2183    LLT Src1Ty = MRI->getType(MI->getOperand(1).getReg());2184    LLT Src2Ty = MRI->getType(MI->getOperand(2).getReg());2185    if (Src1Ty.isVector() != Src2Ty.isVector()) {2186      report("Shifts and rotates require operands to be either all scalars or "2187             "all vectors",2188             MI);2189      break;2190    }2191    break;2192  }2193  case TargetOpcode::G_LLROUND:2194  case TargetOpcode::G_LROUND: {2195    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());2196    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());2197    if (!DstTy.isValid() || !SrcTy.isValid())2198      break;2199    if (SrcTy.isPointer() || DstTy.isPointer()) {2200      StringRef Op = SrcTy.isPointer() ? "Source" : "Destination";2201      report(Twine(Op, " operand must not be a pointer type"), MI);2202    } else if (SrcTy.isScalar()) {2203      verifyAllRegOpsScalar(*MI, *MRI);2204      break;2205    } else if (SrcTy.isVector()) {2206      verifyVectorElementMatch(SrcTy, DstTy, MI);2207      break;2208    }2209    break;2210  }2211  case TargetOpcode::G_IS_FPCLASS: {2212    LLT DestTy = MRI->getType(MI->getOperand(0).getReg());2213    LLT DestEltTy = DestTy.getScalarType();2214    if (!DestEltTy.isScalar()) {2215      report("Destination must be a scalar or vector of scalars", MI);2216      break;2217    }2218    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());2219    LLT SrcEltTy = SrcTy.getScalarType();2220    if (!SrcEltTy.isScalar()) {2221      report("Source must be a scalar or vector of scalars", MI);2222      break;2223    }2224    if (!verifyVectorElementMatch(DestTy, SrcTy, MI))2225      break;2226    const MachineOperand &TestMO = MI->getOperand(2);2227    if (!TestMO.isImm()) {2228      report("floating-point class set (operand 2) must be an immediate", MI);2229      break;2230    }2231    int64_t Test = TestMO.getImm();2232    if (Test < 0 || Test > fcAllFlags) {2233      report("Incorrect floating-point class set (operand 2)", MI);2234      break;2235    }2236    break;2237  }2238  case TargetOpcode::G_PREFETCH: {2239    const MachineOperand &AddrOp = MI->getOperand(0);2240    if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer()) {2241      report("addr operand must be a pointer", &AddrOp, 0);2242      break;2243    }2244    const MachineOperand &RWOp = MI->getOperand(1);2245    if (!RWOp.isImm() || (uint64_t)RWOp.getImm() >= 2) {2246      report("rw operand must be an immediate 0-1", &RWOp, 1);2247      break;2248    }2249    const MachineOperand &LocalityOp = MI->getOperand(2);2250    if (!LocalityOp.isImm() || (uint64_t)LocalityOp.getImm() >= 4) {2251      report("locality operand must be an immediate 0-3", &LocalityOp, 2);2252      break;2253    }2254    const MachineOperand &CacheTypeOp = MI->getOperand(3);2255    if (!CacheTypeOp.isImm() || (uint64_t)CacheTypeOp.getImm() >= 2) {2256      report("cache type operand must be an immediate 0-1", &CacheTypeOp, 3);2257      break;2258    }2259    break;2260  }2261  case TargetOpcode::G_ASSERT_ALIGN: {2262    if (MI->getOperand(2).getImm() < 1)2263      report("alignment immediate must be >= 1", MI);2264    break;2265  }2266  case TargetOpcode::G_CONSTANT_POOL: {2267    if (!MI->getOperand(1).isCPI())2268      report("Src operand 1 must be a constant pool index", MI);2269    if (!MRI->getType(MI->getOperand(0).getReg()).isPointer())2270      report("Dst operand 0 must be a pointer", MI);2271    break;2272  }2273  case TargetOpcode::G_PTRAUTH_GLOBAL_VALUE: {2274    const MachineOperand &AddrOp = MI->getOperand(1);2275    if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer())2276      report("addr operand must be a pointer", &AddrOp, 1);2277    break;2278  }2279  default:2280    break;2281  }2282}2283 2284void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {2285  const MCInstrDesc &MCID = MI->getDesc();2286  if (MI->getNumOperands() < MCID.getNumOperands()) {2287    report("Too few operands", MI);2288    OS << MCID.getNumOperands() << " operands expected, but "2289       << MI->getNumOperands() << " given.\n";2290  }2291 2292  if (MI->getFlag(MachineInstr::NoConvergent) && !MCID.isConvergent())2293    report("NoConvergent flag expected only on convergent instructions.", MI);2294 2295  if (MI->isPHI()) {2296    if (MF->getProperties().hasNoPHIs())2297      report("Found PHI instruction with NoPHIs property set", MI);2298 2299    if (FirstNonPHI)2300      report("Found PHI instruction after non-PHI", MI);2301  } else if (FirstNonPHI == nullptr)2302    FirstNonPHI = MI;2303 2304  // Check the tied operands.2305  if (MI->isInlineAsm())2306    verifyInlineAsm(MI);2307 2308  // Check that unspillable terminators define a reg and have at most one use.2309  if (TII->isUnspillableTerminator(MI)) {2310    if (!MI->getOperand(0).isReg() || !MI->getOperand(0).isDef())2311      report("Unspillable Terminator does not define a reg", MI);2312    Register Def = MI->getOperand(0).getReg();2313    if (Def.isVirtual() && !MF->getProperties().hasNoPHIs() &&2314        std::distance(MRI->use_nodbg_begin(Def), MRI->use_nodbg_end()) > 1)2315      report("Unspillable Terminator expected to have at most one use!", MI);2316  }2317 2318  // A fully-formed DBG_VALUE must have a location. Ignore partially formed2319  // DBG_VALUEs: these are convenient to use in tests, but should never get2320  // generated.2321  if (MI->isDebugValue() && MI->getNumOperands() == 4)2322    if (!MI->getDebugLoc())2323      report("Missing DebugLoc for debug instruction", MI);2324 2325  // Meta instructions should never be the subject of debug value tracking,2326  // they don't create a value in the output program at all.2327  if (MI->isMetaInstruction() && MI->peekDebugInstrNum())2328    report("Metadata instruction should not have a value tracking number", MI);2329 2330  // Check the MachineMemOperands for basic consistency.2331  for (MachineMemOperand *Op : MI->memoperands()) {2332    if (Op->isLoad() && !MI->mayLoad())2333      report("Missing mayLoad flag", MI);2334    if (Op->isStore() && !MI->mayStore())2335      report("Missing mayStore flag", MI);2336  }2337 2338  // Debug values must not have a slot index.2339  // Other instructions must have one, unless they are inside a bundle.2340  if (LiveInts) {2341    bool mapped = !LiveInts->isNotInMIMap(*MI);2342    if (MI->isDebugOrPseudoInstr()) {2343      if (mapped)2344        report("Debug instruction has a slot index", MI);2345    } else if (MI->isInsideBundle()) {2346      if (mapped)2347        report("Instruction inside bundle has a slot index", MI);2348    } else {2349      if (!mapped)2350        report("Missing slot index", MI);2351    }2352  }2353 2354  unsigned Opc = MCID.getOpcode();2355  if (isPreISelGenericOpcode(Opc) || isPreISelGenericOptimizationHint(Opc)) {2356    verifyPreISelGenericInstruction(MI);2357    return;2358  }2359 2360  StringRef ErrorInfo;2361  if (!TII->verifyInstruction(*MI, ErrorInfo))2362    report(ErrorInfo.data(), MI);2363 2364  // Verify properties of various specific instruction types2365  switch (MI->getOpcode()) {2366  case TargetOpcode::COPY: {2367    const MachineOperand &DstOp = MI->getOperand(0);2368    const MachineOperand &SrcOp = MI->getOperand(1);2369    const Register SrcReg = SrcOp.getReg();2370    const Register DstReg = DstOp.getReg();2371 2372    LLT DstTy = MRI->getType(DstReg);2373    LLT SrcTy = MRI->getType(SrcReg);2374    if (SrcTy.isValid() && DstTy.isValid()) {2375      // If both types are valid, check that the types are the same.2376      if (SrcTy != DstTy) {2377        report("Copy Instruction is illegal with mismatching types", MI);2378        OS << "Def = " << DstTy << ", Src = " << SrcTy << '\n';2379      }2380 2381      break;2382    }2383 2384    if (!SrcTy.isValid() && !DstTy.isValid())2385      break;2386 2387    // If we have only one valid type, this is likely a copy between a virtual2388    // and physical register.2389    TypeSize SrcSize = TypeSize::getZero();2390    TypeSize DstSize = TypeSize::getZero();2391    if (SrcReg.isPhysical() && DstTy.isValid()) {2392      const TargetRegisterClass *SrcRC =2393          TRI->getMinimalPhysRegClassLLT(SrcReg, DstTy);2394      if (!SrcRC)2395        SrcSize = TRI->getRegSizeInBits(SrcReg, *MRI);2396    } else {2397      SrcSize = TRI->getRegSizeInBits(SrcReg, *MRI);2398    }2399 2400    if (DstReg.isPhysical() && SrcTy.isValid()) {2401      const TargetRegisterClass *DstRC =2402          TRI->getMinimalPhysRegClassLLT(DstReg, SrcTy);2403      if (!DstRC)2404        DstSize = TRI->getRegSizeInBits(DstReg, *MRI);2405    } else {2406      DstSize = TRI->getRegSizeInBits(DstReg, *MRI);2407    }2408 2409    // The next two checks allow COPY between physical and virtual registers,2410    // when the virtual register has a scalable size and the physical register2411    // has a fixed size. These checks allow COPY between *potentially*2412    // mismatched sizes. However, once RegisterBankSelection occurs,2413    // MachineVerifier should be able to resolve a fixed size for the scalable2414    // vector, and at that point this function will know for sure whether the2415    // sizes are mismatched and correctly report a size mismatch.2416    if (SrcReg.isPhysical() && DstReg.isVirtual() && DstSize.isScalable() &&2417        !SrcSize.isScalable())2418      break;2419    if (SrcReg.isVirtual() && DstReg.isPhysical() && SrcSize.isScalable() &&2420        !DstSize.isScalable())2421      break;2422 2423    if (SrcSize.isNonZero() && DstSize.isNonZero() && SrcSize != DstSize) {2424      if (!DstOp.getSubReg() && !SrcOp.getSubReg()) {2425        report("Copy Instruction is illegal with mismatching sizes", MI);2426        OS << "Def Size = " << DstSize << ", Src Size = " << SrcSize << '\n';2427      }2428    }2429    break;2430  }2431  case TargetOpcode::STATEPOINT: {2432    StatepointOpers SO(MI);2433    if (!MI->getOperand(SO.getIDPos()).isImm() ||2434        !MI->getOperand(SO.getNBytesPos()).isImm() ||2435        !MI->getOperand(SO.getNCallArgsPos()).isImm()) {2436      report("meta operands to STATEPOINT not constant!", MI);2437      break;2438    }2439 2440    auto VerifyStackMapConstant = [&](unsigned Offset) {2441      if (Offset >= MI->getNumOperands()) {2442        report("stack map constant to STATEPOINT is out of range!", MI);2443        return;2444      }2445      if (!MI->getOperand(Offset - 1).isImm() ||2446          MI->getOperand(Offset - 1).getImm() != StackMaps::ConstantOp ||2447          !MI->getOperand(Offset).isImm())2448        report("stack map constant to STATEPOINT not well formed!", MI);2449    };2450    VerifyStackMapConstant(SO.getCCIdx());2451    VerifyStackMapConstant(SO.getFlagsIdx());2452    VerifyStackMapConstant(SO.getNumDeoptArgsIdx());2453    VerifyStackMapConstant(SO.getNumGCPtrIdx());2454    VerifyStackMapConstant(SO.getNumAllocaIdx());2455    VerifyStackMapConstant(SO.getNumGcMapEntriesIdx());2456 2457    // Verify that all explicit statepoint defs are tied to gc operands as2458    // they are expected to be a relocation of gc operands.2459    unsigned FirstGCPtrIdx = SO.getFirstGCPtrIdx();2460    unsigned LastGCPtrIdx = SO.getNumAllocaIdx() - 2;2461    for (unsigned Idx = 0; Idx < MI->getNumDefs(); Idx++) {2462      unsigned UseOpIdx;2463      if (!MI->isRegTiedToUseOperand(Idx, &UseOpIdx)) {2464        report("STATEPOINT defs expected to be tied", MI);2465        break;2466      }2467      if (UseOpIdx < FirstGCPtrIdx || UseOpIdx > LastGCPtrIdx) {2468        report("STATEPOINT def tied to non-gc operand", MI);2469        break;2470      }2471    }2472 2473    // TODO: verify we have properly encoded deopt arguments2474  } break;2475  case TargetOpcode::INSERT_SUBREG: {2476    unsigned InsertedSize;2477    if (unsigned SubIdx = MI->getOperand(2).getSubReg())2478      InsertedSize = TRI->getSubRegIdxSize(SubIdx);2479    else2480      InsertedSize = TRI->getRegSizeInBits(MI->getOperand(2).getReg(), *MRI);2481    unsigned SubRegSize = TRI->getSubRegIdxSize(MI->getOperand(3).getImm());2482    if (SubRegSize < InsertedSize) {2483      report("INSERT_SUBREG expected inserted value to have equal or lesser "2484             "size than the subreg it was inserted into", MI);2485      break;2486    }2487  } break;2488  case TargetOpcode::REG_SEQUENCE: {2489    unsigned NumOps = MI->getNumOperands();2490    if (!(NumOps & 1)) {2491      report("Invalid number of operands for REG_SEQUENCE", MI);2492      break;2493    }2494 2495    for (unsigned I = 1; I != NumOps; I += 2) {2496      const MachineOperand &RegOp = MI->getOperand(I);2497      const MachineOperand &SubRegOp = MI->getOperand(I + 1);2498 2499      if (!RegOp.isReg())2500        report("Invalid register operand for REG_SEQUENCE", &RegOp, I);2501 2502      if (!SubRegOp.isImm() || SubRegOp.getImm() == 0 ||2503          SubRegOp.getImm() >= TRI->getNumSubRegIndices()) {2504        report("Invalid subregister index operand for REG_SEQUENCE",2505               &SubRegOp, I + 1);2506      }2507    }2508 2509    Register DstReg = MI->getOperand(0).getReg();2510    if (DstReg.isPhysical())2511      report("REG_SEQUENCE does not support physical register results", MI);2512 2513    if (MI->getOperand(0).getSubReg())2514      report("Invalid subreg result for REG_SEQUENCE", MI);2515 2516    break;2517  }2518  }2519}2520 2521void2522MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {2523  const MachineInstr *MI = MO->getParent();2524  const MCInstrDesc &MCID = MI->getDesc();2525  unsigned NumDefs = MCID.getNumDefs();2526  if (MCID.getOpcode() == TargetOpcode::PATCHPOINT)2527    NumDefs = (MONum == 0 && MO->isReg()) ? NumDefs : 0;2528 2529  // The first MCID.NumDefs operands must be explicit register defines2530  if (MONum < NumDefs) {2531    const MCOperandInfo &MCOI = MCID.operands()[MONum];2532    if (!MO->isReg())2533      report("Explicit definition must be a register", MO, MONum);2534    else if (!MO->isDef() && !MCOI.isOptionalDef())2535      report("Explicit definition marked as use", MO, MONum);2536    else if (MO->isImplicit())2537      report("Explicit definition marked as implicit", MO, MONum);2538  } else if (MONum < MCID.getNumOperands()) {2539    const MCOperandInfo &MCOI = MCID.operands()[MONum];2540    // Don't check if it's the last operand in a variadic instruction. See,2541    // e.g., LDM_RET in the arm back end. Check non-variadic operands only.2542    bool IsOptional = MI->isVariadic() && MONum == MCID.getNumOperands() - 1;2543    if (!IsOptional) {2544      if (MO->isReg()) {2545        if (MO->isDef() && !MCOI.isOptionalDef() && !MCID.variadicOpsAreDefs())2546          report("Explicit operand marked as def", MO, MONum);2547        if (MO->isImplicit())2548          report("Explicit operand marked as implicit", MO, MONum);2549      }2550 2551      // Check that an instruction has register operands only as expected.2552      if (MCOI.OperandType == MCOI::OPERAND_REGISTER &&2553          !MO->isReg() && !MO->isFI())2554        report("Expected a register operand.", MO, MONum);2555      if (MO->isReg()) {2556        if (MCOI.OperandType == MCOI::OPERAND_IMMEDIATE ||2557            (MCOI.OperandType == MCOI::OPERAND_PCREL &&2558             !TII->isPCRelRegisterOperandLegal(*MO)))2559          report("Expected a non-register operand.", MO, MONum);2560      }2561    }2562 2563    int TiedTo = MCID.getOperandConstraint(MONum, MCOI::TIED_TO);2564    if (TiedTo != -1) {2565      if (!MO->isReg())2566        report("Tied use must be a register", MO, MONum);2567      else if (!MO->isTied())2568        report("Operand should be tied", MO, MONum);2569      else if (unsigned(TiedTo) != MI->findTiedOperandIdx(MONum))2570        report("Tied def doesn't match MCInstrDesc", MO, MONum);2571      else if (MO->getReg().isPhysical()) {2572        const MachineOperand &MOTied = MI->getOperand(TiedTo);2573        if (!MOTied.isReg())2574          report("Tied counterpart must be a register", &MOTied, TiedTo);2575        else if (MOTied.getReg().isPhysical() &&2576                 MO->getReg() != MOTied.getReg())2577          report("Tied physical registers must match.", &MOTied, TiedTo);2578      }2579    } else if (MO->isReg() && MO->isTied())2580      report("Explicit operand should not be tied", MO, MONum);2581  } else if (!MI->isVariadic()) {2582    // ARM adds %reg0 operands to indicate predicates. We'll allow that.2583    if (!MO->isValidExcessOperand())2584      report("Extra explicit operand on non-variadic instruction", MO, MONum);2585  }2586 2587  // Verify earlyClobber def operand2588  if (MCID.getOperandConstraint(MONum, MCOI::EARLY_CLOBBER) != -1) {2589    if (!MO->isReg())2590      report("Early clobber must be a register", MI);2591    if (!MO->isEarlyClobber())2592      report("Missing earlyClobber flag", MI);2593  }2594 2595  switch (MO->getType()) {2596  case MachineOperand::MO_Register: {2597    // Verify debug flag on debug instructions. Check this first because reg02598    // indicates an undefined debug value.2599    if (MI->isDebugInstr() && MO->isUse()) {2600      if (!MO->isDebug())2601        report("Register operand must be marked debug", MO, MONum);2602    } else if (MO->isDebug()) {2603      report("Register operand must not be marked debug", MO, MONum);2604    }2605 2606    const Register Reg = MO->getReg();2607    if (!Reg)2608      return;2609    if (MRI->tracksLiveness() && !MI->isDebugInstr())2610      checkLiveness(MO, MONum);2611 2612    if (MO->isDef() && MO->isUndef() && !MO->getSubReg() &&2613        MO->getReg().isVirtual()) // TODO: Apply to physregs too2614      report("Undef virtual register def operands require a subregister", MO, MONum);2615 2616    // Verify the consistency of tied operands.2617    if (MO->isTied()) {2618      unsigned OtherIdx = MI->findTiedOperandIdx(MONum);2619      const MachineOperand &OtherMO = MI->getOperand(OtherIdx);2620      if (!OtherMO.isReg())2621        report("Must be tied to a register", MO, MONum);2622      if (!OtherMO.isTied())2623        report("Missing tie flags on tied operand", MO, MONum);2624      if (MI->findTiedOperandIdx(OtherIdx) != MONum)2625        report("Inconsistent tie links", MO, MONum);2626      if (MONum < MCID.getNumDefs()) {2627        if (OtherIdx < MCID.getNumOperands()) {2628          if (-1 == MCID.getOperandConstraint(OtherIdx, MCOI::TIED_TO))2629            report("Explicit def tied to explicit use without tie constraint",2630                   MO, MONum);2631        } else {2632          if (!OtherMO.isImplicit())2633            report("Explicit def should be tied to implicit use", MO, MONum);2634        }2635      }2636    }2637 2638    // Verify two-address constraints after the twoaddressinstruction pass.2639    // Both twoaddressinstruction pass and phi-node-elimination pass call2640    // MRI->leaveSSA() to set MF as not IsSSA, we should do the verification2641    // after twoaddressinstruction pass not after phi-node-elimination pass. So2642    // we shouldn't use the IsSSA as the condition, we should based on2643    // TiedOpsRewritten property to verify two-address constraints, this2644    // property will be set in twoaddressinstruction pass.2645    unsigned DefIdx;2646    if (MF->getProperties().hasTiedOpsRewritten() && MO->isUse() &&2647        MI->isRegTiedToDefOperand(MONum, &DefIdx) &&2648        Reg != MI->getOperand(DefIdx).getReg())2649      report("Two-address instruction operands must be identical", MO, MONum);2650 2651    // Check register classes.2652    unsigned SubIdx = MO->getSubReg();2653 2654    if (Reg.isPhysical()) {2655      if (SubIdx) {2656        report("Illegal subregister index for physical register", MO, MONum);2657        return;2658      }2659      if (MONum < MCID.getNumOperands()) {2660        if (const TargetRegisterClass *DRC = TII->getRegClass(MCID, MONum)) {2661          if (!DRC->contains(Reg)) {2662            report("Illegal physical register for instruction", MO, MONum);2663            OS << printReg(Reg, TRI) << " is not a "2664               << TRI->getRegClassName(DRC) << " register.\n";2665          }2666        }2667      }2668      if (MO->isRenamable()) {2669        if (MRI->isReserved(Reg)) {2670          report("isRenamable set on reserved register", MO, MONum);2671          return;2672        }2673      }2674    } else {2675      // Virtual register.2676      const TargetRegisterClass *RC = MRI->getRegClassOrNull(Reg);2677      if (!RC) {2678        // This is a generic virtual register.2679 2680        // Do not allow undef uses for generic virtual registers. This ensures2681        // getVRegDef can never fail and return null on a generic register.2682        //2683        // FIXME: This restriction should probably be broadened to all SSA2684        // MIR. However, DetectDeadLanes/ProcessImplicitDefs technically still2685        // run on the SSA function just before phi elimination.2686        if (MO->isUndef())2687          report("Generic virtual register use cannot be undef", MO, MONum);2688 2689        // Debug value instruction is permitted to use undefined vregs.2690        // This is a performance measure to skip the overhead of immediately2691        // pruning unused debug operands. The final undef substitution occurs2692        // when debug values are allocated in LDVImpl::handleDebugValue, so2693        // these verifications always apply after this pass.2694        if (isFunctionTracksDebugUserValues || !MO->isUse() ||2695            !MI->isDebugValue() || !MRI->def_empty(Reg)) {2696          // If we're post-Select, we can't have gvregs anymore.2697          if (isFunctionSelected) {2698            report("Generic virtual register invalid in a Selected function",2699                   MO, MONum);2700            return;2701          }2702 2703          // The gvreg must have a type and it must not have a SubIdx.2704          LLT Ty = MRI->getType(Reg);2705          if (!Ty.isValid()) {2706            report("Generic virtual register must have a valid type", MO,2707                   MONum);2708            return;2709          }2710 2711          const RegisterBank *RegBank = MRI->getRegBankOrNull(Reg);2712          const RegisterBankInfo *RBI = MF->getSubtarget().getRegBankInfo();2713 2714          // If we're post-RegBankSelect, the gvreg must have a bank.2715          if (!RegBank && isFunctionRegBankSelected) {2716            report("Generic virtual register must have a bank in a "2717                   "RegBankSelected function",2718                   MO, MONum);2719            return;2720          }2721 2722          // Make sure the register fits into its register bank if any.2723          if (RegBank && Ty.isValid() && !Ty.isScalableVector() &&2724              RBI->getMaximumSize(RegBank->getID()) < Ty.getSizeInBits()) {2725            report("Register bank is too small for virtual register", MO,2726                   MONum);2727            OS << "Register bank " << RegBank->getName() << " too small("2728               << RBI->getMaximumSize(RegBank->getID()) << ") to fit "2729               << Ty.getSizeInBits() << "-bits\n";2730            return;2731          }2732        }2733 2734        if (SubIdx)  {2735          report("Generic virtual register does not allow subregister index", MO,2736                 MONum);2737          return;2738        }2739 2740        // If this is a target specific instruction and this operand2741        // has register class constraint, the virtual register must2742        // comply to it.2743        if (!isPreISelGenericOpcode(MCID.getOpcode()) &&2744            MONum < MCID.getNumOperands() && TII->getRegClass(MCID, MONum)) {2745          report("Virtual register does not match instruction constraint", MO,2746                 MONum);2747          OS << "Expect register class "2748             << TRI->getRegClassName(TII->getRegClass(MCID, MONum))2749             << " but got nothing\n";2750          return;2751        }2752 2753        break;2754      }2755      if (SubIdx) {2756        const TargetRegisterClass *SRC =2757          TRI->getSubClassWithSubReg(RC, SubIdx);2758        if (!SRC) {2759          report("Invalid subregister index for virtual register", MO, MONum);2760          OS << "Register class " << TRI->getRegClassName(RC)2761             << " does not support subreg index "2762             << TRI->getSubRegIndexName(SubIdx) << '\n';2763          return;2764        }2765        if (RC != SRC) {2766          report("Invalid register class for subregister index", MO, MONum);2767          OS << "Register class " << TRI->getRegClassName(RC)2768             << " does not fully support subreg index "2769             << TRI->getSubRegIndexName(SubIdx) << '\n';2770          return;2771        }2772      }2773      if (MONum < MCID.getNumOperands()) {2774        if (const TargetRegisterClass *DRC = TII->getRegClass(MCID, MONum)) {2775          if (SubIdx) {2776            const TargetRegisterClass *SuperRC =2777                TRI->getLargestLegalSuperClass(RC, *MF);2778            if (!SuperRC) {2779              report("No largest legal super class exists.", MO, MONum);2780              return;2781            }2782            DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx);2783            if (!DRC) {2784              report("No matching super-reg register class.", MO, MONum);2785              return;2786            }2787          }2788          if (!RC->hasSuperClassEq(DRC)) {2789            report("Illegal virtual register for instruction", MO, MONum);2790            OS << "Expected a " << TRI->getRegClassName(DRC)2791               << " register, but got a " << TRI->getRegClassName(RC)2792               << " register\n";2793          }2794        }2795      }2796    }2797    break;2798  }2799 2800  case MachineOperand::MO_RegisterMask:2801    regMasks.push_back(MO->getRegMask());2802    break;2803 2804  case MachineOperand::MO_MachineBasicBlock:2805    if (MI->isPHI() && !MO->getMBB()->isSuccessor(MI->getParent()))2806      report("PHI operand is not in the CFG", MO, MONum);2807    break;2808 2809  case MachineOperand::MO_FrameIndex:2810    if (LiveStks && LiveStks->hasInterval(MO->getIndex()) &&2811        LiveInts && !LiveInts->isNotInMIMap(*MI)) {2812      int FI = MO->getIndex();2813      LiveInterval &LI = LiveStks->getInterval(FI);2814      SlotIndex Idx = LiveInts->getInstructionIndex(*MI);2815 2816      bool stores = MI->mayStore();2817      bool loads = MI->mayLoad();2818      // For a memory-to-memory move, we need to check if the frame2819      // index is used for storing or loading, by inspecting the2820      // memory operands.2821      if (stores && loads) {2822        for (auto *MMO : MI->memoperands()) {2823          const PseudoSourceValue *PSV = MMO->getPseudoValue();2824          if (PSV == nullptr) continue;2825          const FixedStackPseudoSourceValue *Value =2826            dyn_cast<FixedStackPseudoSourceValue>(PSV);2827          if (Value == nullptr) continue;2828          if (Value->getFrameIndex() != FI) continue;2829 2830          if (MMO->isStore())2831            loads = false;2832          else2833            stores = false;2834          break;2835        }2836        if (loads == stores)2837          report("Missing fixed stack memoperand.", MI);2838      }2839      if (loads && !LI.liveAt(Idx.getRegSlot(true))) {2840        report("Instruction loads from dead spill slot", MO, MONum);2841        OS << "Live stack: " << LI << '\n';2842      }2843      if (stores && !LI.liveAt(Idx.getRegSlot())) {2844        report("Instruction stores to dead spill slot", MO, MONum);2845        OS << "Live stack: " << LI << '\n';2846      }2847    }2848    break;2849 2850  case MachineOperand::MO_CFIIndex:2851    if (MO->getCFIIndex() >= MF->getFrameInstructions().size())2852      report("CFI instruction has invalid index", MO, MONum);2853    break;2854 2855  default:2856    break;2857  }2858}2859 2860void MachineVerifier::checkLivenessAtUse(const MachineOperand *MO,2861                                         unsigned MONum, SlotIndex UseIdx,2862                                         const LiveRange &LR,2863                                         VirtRegOrUnit VRegOrUnit,2864                                         LaneBitmask LaneMask) {2865  const MachineInstr *MI = MO->getParent();2866 2867  if (!LR.verify()) {2868    report("invalid live range", MO, MONum);2869    report_context_liverange(LR);2870    report_context_vreg_regunit(VRegOrUnit);2871    report_context(UseIdx);2872    return;2873  }2874 2875  LiveQueryResult LRQ = LR.Query(UseIdx);2876  bool HasValue = LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut());2877  // Check if we have a segment at the use, note however that we only need one2878  // live subregister range, the others may be dead.2879  if (!HasValue && LaneMask.none()) {2880    report("No live segment at use", MO, MONum);2881    report_context_liverange(LR);2882    report_context_vreg_regunit(VRegOrUnit);2883    report_context(UseIdx);2884  }2885  if (MO->isKill() && !LRQ.isKill()) {2886    report("Live range continues after kill flag", MO, MONum);2887    report_context_liverange(LR);2888    report_context_vreg_regunit(VRegOrUnit);2889    if (LaneMask.any())2890      report_context_lanemask(LaneMask);2891    report_context(UseIdx);2892  }2893}2894 2895void MachineVerifier::checkLivenessAtDef(const MachineOperand *MO,2896                                         unsigned MONum, SlotIndex DefIdx,2897                                         const LiveRange &LR,2898                                         VirtRegOrUnit VRegOrUnit,2899                                         bool SubRangeCheck,2900                                         LaneBitmask LaneMask) {2901  if (!LR.verify()) {2902    report("invalid live range", MO, MONum);2903    report_context_liverange(LR);2904    report_context_vreg_regunit(VRegOrUnit);2905    if (LaneMask.any())2906      report_context_lanemask(LaneMask);2907    report_context(DefIdx);2908  }2909 2910  if (const VNInfo *VNI = LR.getVNInfoAt(DefIdx)) {2911    // The LR can correspond to the whole reg and its def slot is not obliged2912    // to be the same as the MO' def slot. E.g. when we check here "normal"2913    // subreg MO but there is other EC subreg MO in the same instruction so the2914    // whole reg has EC def slot and differs from the currently checked MO' def2915    // slot. For example:2916    // %0 [16e,32r:0) 0@16e  L..3 [16e,32r:0) 0@16e  L..C [16r,32r:0) 0@16r2917    // Check that there is an early-clobber def of the same superregister2918    // somewhere is performed in visitMachineFunctionAfter()2919    if (((SubRangeCheck || MO->getSubReg() == 0) && VNI->def != DefIdx) ||2920        !SlotIndex::isSameInstr(VNI->def, DefIdx) ||2921        (VNI->def != DefIdx &&2922         (!VNI->def.isEarlyClobber() || !DefIdx.isRegister()))) {2923      report("Inconsistent valno->def", MO, MONum);2924      report_context_liverange(LR);2925      report_context_vreg_regunit(VRegOrUnit);2926      if (LaneMask.any())2927        report_context_lanemask(LaneMask);2928      report_context(*VNI);2929      report_context(DefIdx);2930    }2931  } else {2932    report("No live segment at def", MO, MONum);2933    report_context_liverange(LR);2934    report_context_vreg_regunit(VRegOrUnit);2935    if (LaneMask.any())2936      report_context_lanemask(LaneMask);2937    report_context(DefIdx);2938  }2939  // Check that, if the dead def flag is present, LiveInts agree.2940  if (MO->isDead()) {2941    LiveQueryResult LRQ = LR.Query(DefIdx);2942    if (!LRQ.isDeadDef()) {2943      assert(VRegOrUnit.isVirtualReg() && "Expecting a virtual register.");2944      // A dead subreg def only tells us that the specific subreg is dead. There2945      // could be other non-dead defs of other subregs, or we could have other2946      // parts of the register being live through the instruction. So unless we2947      // are checking liveness for a subrange it is ok for the live range to2948      // continue, given that we have a dead def of a subregister.2949      if (SubRangeCheck || MO->getSubReg() == 0) {2950        report("Live range continues after dead def flag", MO, MONum);2951        report_context_liverange(LR);2952        report_context_vreg_regunit(VRegOrUnit);2953        if (LaneMask.any())2954          report_context_lanemask(LaneMask);2955      }2956    }2957  }2958}2959 2960void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) {2961  const MachineInstr *MI = MO->getParent();2962  const Register Reg = MO->getReg();2963  const unsigned SubRegIdx = MO->getSubReg();2964 2965  const LiveInterval *LI = nullptr;2966  if (LiveInts && Reg.isVirtual()) {2967    if (LiveInts->hasInterval(Reg)) {2968      LI = &LiveInts->getInterval(Reg);2969      if (SubRegIdx != 0 && (MO->isDef() || !MO->isUndef()) && !LI->empty() &&2970          !LI->hasSubRanges() && MRI->shouldTrackSubRegLiveness(Reg))2971        report("Live interval for subreg operand has no subranges", MO, MONum);2972    } else {2973      report("Virtual register has no live interval", MO, MONum);2974    }2975  }2976 2977  // Both use and def operands can read a register.2978  if (MO->readsReg()) {2979    if (MO->isKill())2980      addRegWithSubRegs(regsKilled, Reg);2981 2982    // Check that LiveVars knows this kill (unless we are inside a bundle, in2983    // which case we have already checked that LiveVars knows any kills on the2984    // bundle header instead).2985    if (LiveVars && Reg.isVirtual() && MO->isKill() &&2986        !MI->isBundledWithPred()) {2987      LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);2988      if (!is_contained(VI.Kills, MI))2989        report("Kill missing from LiveVariables", MO, MONum);2990    }2991 2992    // Check LiveInts liveness and kill.2993    if (LiveInts && !LiveInts->isNotInMIMap(*MI)) {2994      SlotIndex UseIdx;2995      if (MI->isPHI()) {2996        // PHI use occurs on the edge, so check for live out here instead.2997        UseIdx = LiveInts->getMBBEndIdx(2998          MI->getOperand(MONum + 1).getMBB()).getPrevSlot();2999      } else {3000        UseIdx = LiveInts->getInstructionIndex(*MI);3001      }3002      // Check the cached regunit intervals.3003      if (Reg.isPhysical() && !isReserved(Reg)) {3004        for (MCRegUnit Unit : TRI->regunits(Reg.asMCReg())) {3005          if (MRI->isReservedRegUnit(Unit))3006            continue;3007          if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))3008            checkLivenessAtUse(MO, MONum, UseIdx, *LR, VirtRegOrUnit(Unit));3009        }3010      }3011 3012      if (Reg.isVirtual()) {3013        // This is a virtual register interval.3014        checkLivenessAtUse(MO, MONum, UseIdx, *LI, VirtRegOrUnit(Reg));3015 3016        if (LI->hasSubRanges() && !MO->isDef()) {3017          LaneBitmask MOMask = SubRegIdx != 03018                                   ? TRI->getSubRegIndexLaneMask(SubRegIdx)3019                                   : MRI->getMaxLaneMaskForVReg(Reg);3020          LaneBitmask LiveInMask;3021          for (const LiveInterval::SubRange &SR : LI->subranges()) {3022            if ((MOMask & SR.LaneMask).none())3023              continue;3024            checkLivenessAtUse(MO, MONum, UseIdx, SR, VirtRegOrUnit(Reg),3025                               SR.LaneMask);3026            LiveQueryResult LRQ = SR.Query(UseIdx);3027            if (LRQ.valueIn() || (MI->isPHI() && LRQ.valueOut()))3028              LiveInMask |= SR.LaneMask;3029          }3030          // At least parts of the register has to be live at the use.3031          if ((LiveInMask & MOMask).none()) {3032            report("No live subrange at use", MO, MONum);3033            report_context(*LI);3034            report_context(UseIdx);3035          }3036          // For PHIs all lanes should be live3037          if (MI->isPHI() && LiveInMask != MOMask) {3038            report("Not all lanes of PHI source live at use", MO, MONum);3039            report_context(*LI);3040            report_context(UseIdx);3041          }3042        }3043      }3044    }3045 3046    // Use of a dead register.3047    if (!regsLive.count(Reg)) {3048      if (Reg.isPhysical()) {3049        // Reserved registers may be used even when 'dead'.3050        bool Bad = !isReserved(Reg);3051        // We are fine if just any subregister has a defined value.3052        if (Bad) {3053 3054          for (const MCPhysReg &SubReg : TRI->subregs(Reg)) {3055            if (regsLive.count(SubReg)) {3056              Bad = false;3057              break;3058            }3059          }3060        }3061        // If there is an additional implicit-use of a super register we stop3062        // here. By definition we are fine if the super register is not3063        // (completely) dead, if the complete super register is dead we will3064        // get a report for its operand.3065        if (Bad) {3066          for (const MachineOperand &MOP : MI->uses()) {3067            if (!MOP.isReg() || !MOP.isImplicit())3068              continue;3069 3070            if (!MOP.getReg().isPhysical())3071              continue;3072 3073            if (MOP.getReg() != Reg &&3074                all_of(TRI->regunits(Reg), [&](const MCRegUnit RegUnit) {3075                  return llvm::is_contained(TRI->regunits(MOP.getReg()),3076                                            RegUnit);3077                }))3078              Bad = false;3079          }3080        }3081        if (Bad)3082          report("Using an undefined physical register", MO, MONum);3083      } else if (MRI->def_empty(Reg)) {3084        report("Reading virtual register without a def", MO, MONum);3085      } else {3086        BBInfo &MInfo = MBBInfoMap[MI->getParent()];3087        // We don't know which virtual registers are live in, so only complain3088        // if vreg was killed in this MBB. Otherwise keep track of vregs that3089        // must be live in. PHI instructions are handled separately.3090        if (MInfo.regsKilled.count(Reg))3091          report("Using a killed virtual register", MO, MONum);3092        else if (!MI->isPHI())3093          MInfo.vregsLiveIn.insert(std::make_pair(Reg, MI));3094      }3095    }3096  }3097 3098  if (MO->isDef()) {3099    // Register defined.3100    // TODO: verify that earlyclobber ops are not used.3101    if (MO->isDead())3102      addRegWithSubRegs(regsDead, Reg);3103    else3104      addRegWithSubRegs(regsDefined, Reg);3105 3106    // Verify SSA form.3107    if (MRI->isSSA() && Reg.isVirtual() &&3108        std::next(MRI->def_begin(Reg)) != MRI->def_end())3109      report("Multiple virtual register defs in SSA form", MO, MONum);3110 3111    // Check LiveInts for a live segment, but only for virtual registers.3112    if (LiveInts && !LiveInts->isNotInMIMap(*MI)) {3113      SlotIndex DefIdx = LiveInts->getInstructionIndex(*MI);3114      DefIdx = DefIdx.getRegSlot(MO->isEarlyClobber());3115 3116      if (Reg.isVirtual()) {3117        checkLivenessAtDef(MO, MONum, DefIdx, *LI, VirtRegOrUnit(Reg));3118 3119        if (LI->hasSubRanges()) {3120          LaneBitmask MOMask = SubRegIdx != 03121                                   ? TRI->getSubRegIndexLaneMask(SubRegIdx)3122                                   : MRI->getMaxLaneMaskForVReg(Reg);3123          for (const LiveInterval::SubRange &SR : LI->subranges()) {3124            if ((SR.LaneMask & MOMask).none())3125              continue;3126            checkLivenessAtDef(MO, MONum, DefIdx, SR, VirtRegOrUnit(Reg), true,3127                               SR.LaneMask);3128          }3129        }3130      }3131    }3132  }3133}3134 3135// This function gets called after visiting all instructions in a bundle. The3136// argument points to the bundle header.3137// Normal stand-alone instructions are also considered 'bundles', and this3138// function is called for all of them.3139void MachineVerifier::visitMachineBundleAfter(const MachineInstr *MI) {3140  BBInfo &MInfo = MBBInfoMap[MI->getParent()];3141  set_union(MInfo.regsKilled, regsKilled);3142  set_subtract(regsLive, regsKilled); regsKilled.clear();3143  // Kill any masked registers.3144  while (!regMasks.empty()) {3145    const uint32_t *Mask = regMasks.pop_back_val();3146    for (Register Reg : regsLive)3147      if (Reg.isPhysical() &&3148          MachineOperand::clobbersPhysReg(Mask, Reg.asMCReg()))3149        regsDead.push_back(Reg);3150  }3151  set_subtract(regsLive, regsDead);   regsDead.clear();3152  set_union(regsLive, regsDefined);   regsDefined.clear();3153}3154 3155void3156MachineVerifier::visitMachineBasicBlockAfter(const MachineBasicBlock *MBB) {3157  MBBInfoMap[MBB].regsLiveOut = regsLive;3158  regsLive.clear();3159 3160  if (Indexes) {3161    SlotIndex stop = Indexes->getMBBEndIdx(MBB);3162    if (!(stop > lastIndex)) {3163      report("Block ends before last instruction index", MBB);3164      OS << "Block ends at " << stop << " last instruction was at " << lastIndex3165         << '\n';3166    }3167    lastIndex = stop;3168  }3169}3170 3171namespace {3172// This implements a set of registers that serves as a filter: can filter other3173// sets by passing through elements not in the filter and blocking those that3174// are. Any filter implicitly includes the full set of physical registers upon3175// creation, thus filtering them all out. The filter itself as a set only grows,3176// and needs to be as efficient as possible.3177struct VRegFilter {3178  // Add elements to the filter itself. \pre Input set \p FromRegSet must have3179  // no duplicates. Both virtual and physical registers are fine.3180  template <typename RegSetT> void add(const RegSetT &FromRegSet) {3181    SmallVector<Register, 0> VRegsBuffer;3182    filterAndAdd(FromRegSet, VRegsBuffer);3183  }3184  // Filter \p FromRegSet through the filter and append passed elements into \p3185  // ToVRegs. All elements appended are then added to the filter itself.3186  // \returns true if anything changed.3187  template <typename RegSetT>3188  bool filterAndAdd(const RegSetT &FromRegSet,3189                    SmallVectorImpl<Register> &ToVRegs) {3190    unsigned SparseUniverse = Sparse.size();3191    unsigned NewSparseUniverse = SparseUniverse;3192    unsigned NewDenseSize = Dense.size();3193    size_t Begin = ToVRegs.size();3194    for (Register Reg : FromRegSet) {3195      if (!Reg.isVirtual())3196        continue;3197      unsigned Index = Reg.virtRegIndex();3198      if (Index < SparseUniverseMax) {3199        if (Index < SparseUniverse && Sparse.test(Index))3200          continue;3201        NewSparseUniverse = std::max(NewSparseUniverse, Index + 1);3202      } else {3203        if (Dense.count(Reg))3204          continue;3205        ++NewDenseSize;3206      }3207      ToVRegs.push_back(Reg);3208    }3209    size_t End = ToVRegs.size();3210    if (Begin == End)3211      return false;3212    // Reserving space in sets once performs better than doing so continuously3213    // and pays easily for double look-ups (even in Dense with SparseUniverseMax3214    // tuned all the way down) and double iteration (the second one is over a3215    // SmallVector, which is a lot cheaper compared to DenseSet or BitVector).3216    Sparse.resize(NewSparseUniverse);3217    Dense.reserve(NewDenseSize);3218    for (unsigned I = Begin; I < End; ++I) {3219      Register Reg = ToVRegs[I];3220      unsigned Index = Reg.virtRegIndex();3221      if (Index < SparseUniverseMax)3222        Sparse.set(Index);3223      else3224        Dense.insert(Reg);3225    }3226    return true;3227  }3228 3229private:3230  static constexpr unsigned SparseUniverseMax = 10 * 1024 * 8;3231  // VRegs indexed within SparseUniverseMax are tracked by Sparse, those beyond3232  // are tracked by Dense. The only purpose of the threshold and the Dense set3233  // is to have a reasonably growing memory usage in pathological cases (large3234  // number of very sparse VRegFilter instances live at the same time). In3235  // practice even in the worst-by-execution time cases having all elements3236  // tracked by Sparse (very large SparseUniverseMax scenario) tends to be more3237  // space efficient than if tracked by Dense. The threshold is set to keep the3238  // worst-case memory usage within 2x of figures determined empirically for3239  // "all Dense" scenario in such worst-by-execution-time cases.3240  BitVector Sparse;3241  DenseSet<Register> Dense;3242};3243 3244// Implements both a transfer function and a (binary, in-place) join operator3245// for a dataflow over register sets with set union join and filtering transfer3246// (out_b = in_b \ filter_b). filter_b is expected to be set-up ahead of time.3247// Maintains out_b as its state, allowing for O(n) iteration over it at any3248// time, where n is the size of the set (as opposed to O(U) where U is the3249// universe). filter_b implicitly contains all physical registers at all times.3250class FilteringVRegSet {3251  VRegFilter Filter;3252  SmallVector<Register, 0> VRegs;3253 3254public:3255  // Set-up the filter_b. \pre Input register set \p RS must have no duplicates.3256  // Both virtual and physical registers are fine.3257  template <typename RegSetT> void addToFilter(const RegSetT &RS) {3258    Filter.add(RS);3259  }3260  // Passes \p RS through the filter_b (transfer function) and adds what's left3261  // to itself (out_b).3262  template <typename RegSetT> bool add(const RegSetT &RS) {3263    // Double-duty the Filter: to maintain VRegs a set (and the join operation3264    // a set union) just add everything being added here to the Filter as well.3265    return Filter.filterAndAdd(RS, VRegs);3266  }3267  using const_iterator = decltype(VRegs)::const_iterator;3268  const_iterator begin() const { return VRegs.begin(); }3269  const_iterator end() const { return VRegs.end(); }3270  size_t size() const { return VRegs.size(); }3271};3272} // namespace3273 3274// Calculate the largest possible vregsPassed sets. These are the registers that3275// can pass through an MBB live, but may not be live every time. It is assumed3276// that all vregsPassed sets are empty before the call.3277void MachineVerifier::calcRegsPassed() {3278  if (MF->empty())3279    // ReversePostOrderTraversal doesn't handle empty functions.3280    return;3281 3282  for (const MachineBasicBlock *MB :3283       ReversePostOrderTraversal<const MachineFunction *>(MF)) {3284    FilteringVRegSet VRegs;3285    BBInfo &Info = MBBInfoMap[MB];3286    assert(Info.reachable);3287 3288    VRegs.addToFilter(Info.regsKilled);3289    VRegs.addToFilter(Info.regsLiveOut);3290    for (const MachineBasicBlock *Pred : MB->predecessors()) {3291      const BBInfo &PredInfo = MBBInfoMap[Pred];3292      if (!PredInfo.reachable)3293        continue;3294 3295      VRegs.add(PredInfo.regsLiveOut);3296      VRegs.add(PredInfo.vregsPassed);3297    }3298    Info.vregsPassed.reserve(VRegs.size());3299    Info.vregsPassed.insert_range(VRegs);3300  }3301}3302 3303// Calculate the set of virtual registers that must be passed through each basic3304// block in order to satisfy the requirements of successor blocks. This is very3305// similar to calcRegsPassed, only backwards.3306void MachineVerifier::calcRegsRequired() {3307  // First push live-in regs to predecessors' vregsRequired.3308  SmallPtrSet<const MachineBasicBlock*, 8> todo;3309  for (const auto &MBB : *MF) {3310    BBInfo &MInfo = MBBInfoMap[&MBB];3311    for (const MachineBasicBlock *Pred : MBB.predecessors()) {3312      BBInfo &PInfo = MBBInfoMap[Pred];3313      if (PInfo.addRequired(MInfo.vregsLiveIn))3314        todo.insert(Pred);3315    }3316 3317    // Handle the PHI node.3318    for (const MachineInstr &MI : MBB.phis()) {3319      for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {3320        // Skip those Operands which are undef regs or not regs.3321        if (!MI.getOperand(i).isReg() || !MI.getOperand(i).readsReg())3322          continue;3323 3324        // Get register and predecessor for one PHI edge.3325        Register Reg = MI.getOperand(i).getReg();3326        const MachineBasicBlock *Pred = MI.getOperand(i + 1).getMBB();3327 3328        BBInfo &PInfo = MBBInfoMap[Pred];3329        if (PInfo.addRequired(Reg))3330          todo.insert(Pred);3331      }3332    }3333  }3334 3335  // Iteratively push vregsRequired to predecessors. This will converge to the3336  // same final state regardless of DenseSet iteration order.3337  while (!todo.empty()) {3338    const MachineBasicBlock *MBB = *todo.begin();3339    todo.erase(MBB);3340    BBInfo &MInfo = MBBInfoMap[MBB];3341    for (const MachineBasicBlock *Pred : MBB->predecessors()) {3342      if (Pred == MBB)3343        continue;3344      BBInfo &SInfo = MBBInfoMap[Pred];3345      if (SInfo.addRequired(MInfo.vregsRequired))3346        todo.insert(Pred);3347    }3348  }3349}3350 3351// Check PHI instructions at the beginning of MBB. It is assumed that3352// calcRegsPassed has been run so BBInfo::isLiveOut is valid.3353void MachineVerifier::checkPHIOps(const MachineBasicBlock &MBB) {3354  BBInfo &MInfo = MBBInfoMap[&MBB];3355 3356  SmallPtrSet<const MachineBasicBlock*, 8> seen;3357  for (const MachineInstr &Phi : MBB) {3358    if (!Phi.isPHI())3359      break;3360    seen.clear();3361 3362    const MachineOperand &MODef = Phi.getOperand(0);3363    if (!MODef.isReg() || !MODef.isDef()) {3364      report("Expected first PHI operand to be a register def", &MODef, 0);3365      continue;3366    }3367    if (MODef.isTied() || MODef.isImplicit() || MODef.isInternalRead() ||3368        MODef.isEarlyClobber() || MODef.isDebug())3369      report("Unexpected flag on PHI operand", &MODef, 0);3370    Register DefReg = MODef.getReg();3371    if (!DefReg.isVirtual())3372      report("Expected first PHI operand to be a virtual register", &MODef, 0);3373 3374    for (unsigned I = 1, E = Phi.getNumOperands(); I != E; I += 2) {3375      const MachineOperand &MO0 = Phi.getOperand(I);3376      if (!MO0.isReg()) {3377        report("Expected PHI operand to be a register", &MO0, I);3378        continue;3379      }3380      if (MO0.isImplicit() || MO0.isInternalRead() || MO0.isEarlyClobber() ||3381          MO0.isDebug() || MO0.isTied())3382        report("Unexpected flag on PHI operand", &MO0, I);3383 3384      const MachineOperand &MO1 = Phi.getOperand(I + 1);3385      if (!MO1.isMBB()) {3386        report("Expected PHI operand to be a basic block", &MO1, I + 1);3387        continue;3388      }3389 3390      const MachineBasicBlock &Pre = *MO1.getMBB();3391      if (!Pre.isSuccessor(&MBB)) {3392        report("PHI input is not a predecessor block", &MO1, I + 1);3393        continue;3394      }3395 3396      if (MInfo.reachable) {3397        seen.insert(&Pre);3398        BBInfo &PrInfo = MBBInfoMap[&Pre];3399        if (!MO0.isUndef() && PrInfo.reachable &&3400            !PrInfo.isLiveOut(MO0.getReg()))3401          report("PHI operand is not live-out from predecessor", &MO0, I);3402      }3403    }3404 3405    // Did we see all predecessors?3406    if (MInfo.reachable) {3407      for (MachineBasicBlock *Pred : MBB.predecessors()) {3408        if (!seen.count(Pred)) {3409          report("Missing PHI operand", &Phi);3410          OS << printMBBReference(*Pred)3411             << " is a predecessor according to the CFG.\n";3412        }3413      }3414    }3415  }3416}3417 3418static void3419verifyConvergenceControl(const MachineFunction &MF, MachineDominatorTree &DT,3420                         std::function<void(const Twine &Message)> FailureCB,3421                         raw_ostream &OS) {3422  MachineConvergenceVerifier CV;3423  CV.initialize(&OS, FailureCB, MF);3424 3425  for (const auto &MBB : MF) {3426    CV.visit(MBB);3427    for (const auto &MI : MBB.instrs())3428      CV.visit(MI);3429  }3430 3431  if (CV.sawTokens()) {3432    DT.recalculate(const_cast<MachineFunction &>(MF));3433    CV.verify(DT);3434  }3435}3436 3437void MachineVerifier::visitMachineFunctionAfter() {3438  auto FailureCB = [this](const Twine &Message) {3439    report(Message.str().c_str(), MF);3440  };3441  verifyConvergenceControl(*MF, DT, FailureCB, OS);3442 3443  calcRegsPassed();3444 3445  for (const MachineBasicBlock &MBB : *MF)3446    checkPHIOps(MBB);3447 3448  // Now check liveness info if available3449  calcRegsRequired();3450 3451  // Check for killed virtual registers that should be live out.3452  for (const auto &MBB : *MF) {3453    BBInfo &MInfo = MBBInfoMap[&MBB];3454    for (Register VReg : MInfo.vregsRequired)3455      if (MInfo.regsKilled.count(VReg)) {3456        report("Virtual register killed in block, but needed live out.", &MBB);3457        OS << "Virtual register " << printReg(VReg)3458           << " is used after the block.\n";3459      }3460  }3461 3462  if (!MF->empty()) {3463    BBInfo &MInfo = MBBInfoMap[&MF->front()];3464    for (Register VReg : MInfo.vregsRequired) {3465      report("Virtual register defs don't dominate all uses.", MF);3466      report_context_vreg(VReg);3467    }3468  }3469 3470  if (LiveVars)3471    verifyLiveVariables();3472  if (LiveInts)3473    verifyLiveIntervals();3474 3475  // Check live-in list of each MBB. If a register is live into MBB, check3476  // that the register is in regsLiveOut of each predecessor block. Since3477  // this must come from a definition in the predecessor or its live-in3478  // list, this will catch a live-through case where the predecessor does not3479  // have the register in its live-in list.  This currently only checks3480  // registers that have no aliases, are not allocatable and are not3481  // reserved, which could mean a condition code register for instance.3482  if (MRI->tracksLiveness())3483    for (const auto &MBB : *MF)3484      for (MachineBasicBlock::RegisterMaskPair P : MBB.liveins()) {3485        MCRegister LiveInReg = P.PhysReg;3486        bool hasAliases = MCRegAliasIterator(LiveInReg, TRI, false).isValid();3487        if (hasAliases || isAllocatable(LiveInReg) || isReserved(LiveInReg))3488          continue;3489        for (const MachineBasicBlock *Pred : MBB.predecessors()) {3490          BBInfo &PInfo = MBBInfoMap[Pred];3491          if (!PInfo.regsLiveOut.count(LiveInReg)) {3492            report("Live in register not found to be live out from predecessor.",3493                   &MBB);3494            OS << TRI->getName(LiveInReg) << " not found to be live out from "3495               << printMBBReference(*Pred) << '\n';3496          }3497        }3498      }3499 3500  for (auto CSInfo : MF->getCallSitesInfo())3501    if (!CSInfo.first->isCall())3502      report("Call site info referencing instruction that is not call", MF);3503 3504  // If there's debug-info, check that we don't have any duplicate value3505  // tracking numbers.3506  if (MF->getFunction().getSubprogram()) {3507    DenseSet<unsigned> SeenNumbers;3508    for (const auto &MBB : *MF) {3509      for (const auto &MI : MBB) {3510        if (auto Num = MI.peekDebugInstrNum()) {3511          auto Result = SeenNumbers.insert((unsigned)Num);3512          if (!Result.second)3513            report("Instruction has a duplicated value tracking number", &MI);3514        }3515      }3516    }3517  }3518}3519 3520void MachineVerifier::verifyLiveVariables() {3521  assert(LiveVars && "Don't call verifyLiveVariables without LiveVars");3522  for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {3523    Register Reg = Register::index2VirtReg(I);3524    LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);3525    for (const auto &MBB : *MF) {3526      BBInfo &MInfo = MBBInfoMap[&MBB];3527 3528      // Our vregsRequired should be identical to LiveVariables' AliveBlocks3529      if (MInfo.vregsRequired.count(Reg)) {3530        if (!VI.AliveBlocks.test(MBB.getNumber())) {3531          report("LiveVariables: Block missing from AliveBlocks", &MBB);3532          OS << "Virtual register " << printReg(Reg)3533             << " must be live through the block.\n";3534        }3535      } else {3536        if (VI.AliveBlocks.test(MBB.getNumber())) {3537          report("LiveVariables: Block should not be in AliveBlocks", &MBB);3538          OS << "Virtual register " << printReg(Reg)3539             << " is not needed live through the block.\n";3540        }3541      }3542    }3543  }3544}3545 3546void MachineVerifier::verifyLiveIntervals() {3547  assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts");3548  for (unsigned I = 0, E = MRI->getNumVirtRegs(); I != E; ++I) {3549    Register Reg = Register::index2VirtReg(I);3550 3551    // Spilling and splitting may leave unused registers around. Skip them.3552    if (MRI->reg_nodbg_empty(Reg))3553      continue;3554 3555    if (!LiveInts->hasInterval(Reg)) {3556      report("Missing live interval for virtual register", MF);3557      OS << printReg(Reg, TRI) << " still has defs or uses\n";3558      continue;3559    }3560 3561    const LiveInterval &LI = LiveInts->getInterval(Reg);3562    assert(Reg == LI.reg() && "Invalid reg to interval mapping");3563    verifyLiveInterval(LI);3564  }3565 3566  // Verify all the cached regunit intervals.3567  for (MCRegUnit Unit : TRI->regunits())3568    if (const LiveRange *LR = LiveInts->getCachedRegUnit(Unit))3569      verifyLiveRange(*LR, VirtRegOrUnit(Unit));3570}3571 3572void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR,3573                                           const VNInfo *VNI,3574                                           VirtRegOrUnit VRegOrUnit,3575                                           LaneBitmask LaneMask) {3576  if (VNI->isUnused())3577    return;3578 3579  const VNInfo *DefVNI = LR.getVNInfoAt(VNI->def);3580 3581  if (!DefVNI) {3582    report("Value not live at VNInfo def and not marked unused", MF);3583    report_context(LR, VRegOrUnit, LaneMask);3584    report_context(*VNI);3585    return;3586  }3587 3588  if (DefVNI != VNI) {3589    report("Live segment at def has different VNInfo", MF);3590    report_context(LR, VRegOrUnit, LaneMask);3591    report_context(*VNI);3592    return;3593  }3594 3595  const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(VNI->def);3596  if (!MBB) {3597    report("Invalid VNInfo definition index", MF);3598    report_context(LR, VRegOrUnit, LaneMask);3599    report_context(*VNI);3600    return;3601  }3602 3603  if (VNI->isPHIDef()) {3604    if (VNI->def != LiveInts->getMBBStartIdx(MBB)) {3605      report("PHIDef VNInfo is not defined at MBB start", MBB);3606      report_context(LR, VRegOrUnit, LaneMask);3607      report_context(*VNI);3608    }3609    return;3610  }3611 3612  // Non-PHI def.3613  const MachineInstr *MI = LiveInts->getInstructionFromIndex(VNI->def);3614  if (!MI) {3615    report("No instruction at VNInfo def index", MBB);3616    report_context(LR, VRegOrUnit, LaneMask);3617    report_context(*VNI);3618    return;3619  }3620 3621  bool hasDef = false;3622  bool isEarlyClobber = false;3623  for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {3624    if (!MOI->isReg() || !MOI->isDef())3625      continue;3626    if (VRegOrUnit.isVirtualReg()) {3627      if (MOI->getReg() != VRegOrUnit.asVirtualReg())3628        continue;3629    } else {3630      if (!MOI->getReg().isPhysical() ||3631          !TRI->hasRegUnit(MOI->getReg(), VRegOrUnit.asMCRegUnit()))3632        continue;3633    }3634    if (LaneMask.any() &&3635        (TRI->getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask).none())3636      continue;3637    hasDef = true;3638    if (MOI->isEarlyClobber())3639      isEarlyClobber = true;3640  }3641 3642  if (!hasDef) {3643    report("Defining instruction does not modify register", MI);3644    report_context(LR, VRegOrUnit, LaneMask);3645    report_context(*VNI);3646  }3647 3648  // Early clobber defs begin at USE slots, but other defs must begin at3649  // DEF slots.3650  if (isEarlyClobber) {3651    if (!VNI->def.isEarlyClobber()) {3652      report("Early clobber def must be at an early-clobber slot", MBB);3653      report_context(LR, VRegOrUnit, LaneMask);3654      report_context(*VNI);3655    }3656  } else if (!VNI->def.isRegister()) {3657    report("Non-PHI, non-early clobber def must be at a register slot", MBB);3658    report_context(LR, VRegOrUnit, LaneMask);3659    report_context(*VNI);3660  }3661}3662 3663void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,3664                                             const LiveRange::const_iterator I,3665                                             VirtRegOrUnit VRegOrUnit,3666                                             LaneBitmask LaneMask) {3667  const LiveRange::Segment &S = *I;3668  const VNInfo *VNI = S.valno;3669  assert(VNI && "Live segment has no valno");3670 3671  if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(VNI->id)) {3672    report("Foreign valno in live segment", MF);3673    report_context(LR, VRegOrUnit, LaneMask);3674    report_context(S);3675    report_context(*VNI);3676  }3677 3678  if (VNI->isUnused()) {3679    report("Live segment valno is marked unused", MF);3680    report_context(LR, VRegOrUnit, LaneMask);3681    report_context(S);3682  }3683 3684  const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(S.start);3685  if (!MBB) {3686    report("Bad start of live segment, no basic block", MF);3687    report_context(LR, VRegOrUnit, LaneMask);3688    report_context(S);3689    return;3690  }3691  SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(MBB);3692  if (S.start != MBBStartIdx && S.start != VNI->def) {3693    report("Live segment must begin at MBB entry or valno def", MBB);3694    report_context(LR, VRegOrUnit, LaneMask);3695    report_context(S);3696  }3697 3698  const MachineBasicBlock *EndMBB =3699    LiveInts->getMBBFromIndex(S.end.getPrevSlot());3700  if (!EndMBB) {3701    report("Bad end of live segment, no basic block", MF);3702    report_context(LR, VRegOrUnit, LaneMask);3703    report_context(S);3704    return;3705  }3706 3707  // Checks for non-live-out segments.3708  if (S.end != LiveInts->getMBBEndIdx(EndMBB)) {3709    // RegUnit intervals are allowed dead phis.3710    if (!VRegOrUnit.isVirtualReg() && VNI->isPHIDef() && S.start == VNI->def &&3711        S.end == VNI->def.getDeadSlot())3712      return;3713 3714    // The live segment is ending inside EndMBB3715    const MachineInstr *MI =3716        LiveInts->getInstructionFromIndex(S.end.getPrevSlot());3717    if (!MI) {3718      report("Live segment doesn't end at a valid instruction", EndMBB);3719      report_context(LR, VRegOrUnit, LaneMask);3720      report_context(S);3721      return;3722    }3723 3724    // The block slot must refer to a basic block boundary.3725    if (S.end.isBlock()) {3726      report("Live segment ends at B slot of an instruction", EndMBB);3727      report_context(LR, VRegOrUnit, LaneMask);3728      report_context(S);3729    }3730 3731    if (S.end.isDead()) {3732      // Segment ends on the dead slot.3733      // That means there must be a dead def.3734      if (!SlotIndex::isSameInstr(S.start, S.end)) {3735        report("Live segment ending at dead slot spans instructions", EndMBB);3736        report_context(LR, VRegOrUnit, LaneMask);3737        report_context(S);3738      }3739    }3740 3741    // After tied operands are rewritten, a live segment can only end at an3742    // early-clobber slot if it is being redefined by an early-clobber def.3743    // TODO: Before tied operands are rewritten, a live segment can only end at3744    // an early-clobber slot if the last use is tied to an early-clobber def.3745    if (MF->getProperties().hasTiedOpsRewritten() && S.end.isEarlyClobber()) {3746      if (I + 1 == LR.end() || (I + 1)->start != S.end) {3747        report("Live segment ending at early clobber slot must be "3748               "redefined by an EC def in the same instruction",3749               EndMBB);3750        report_context(LR, VRegOrUnit, LaneMask);3751        report_context(S);3752      }3753    }3754 3755    // The following checks only apply to virtual registers. Physreg liveness3756    // is too weird to check.3757    if (VRegOrUnit.isVirtualReg()) {3758      // A live segment can end with either a redefinition, a kill flag on a3759      // use, or a dead flag on a def.3760      bool hasRead = false;3761      bool hasSubRegDef = false;3762      bool hasDeadDef = false;3763      for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {3764        if (!MOI->isReg() || MOI->getReg() != VRegOrUnit.asVirtualReg())3765          continue;3766        unsigned Sub = MOI->getSubReg();3767        LaneBitmask SLM =3768            Sub != 0 ? TRI->getSubRegIndexLaneMask(Sub) : LaneBitmask::getAll();3769        if (MOI->isDef()) {3770          if (Sub != 0) {3771            hasSubRegDef = true;3772            // An operand %0:sub0 reads %0:sub1..n. Invert the lane3773            // mask for subregister defs. Read-undef defs will be handled by3774            // readsReg below.3775            SLM = ~SLM;3776          }3777          if (MOI->isDead())3778            hasDeadDef = true;3779        }3780        if (LaneMask.any() && (LaneMask & SLM).none())3781          continue;3782        if (MOI->readsReg())3783          hasRead = true;3784      }3785      if (S.end.isDead()) {3786        // Make sure that the corresponding machine operand for a "dead" live3787        // range has the dead flag. We cannot perform this check for subregister3788        // liveranges as partially dead values are allowed.3789        if (LaneMask.none() && !hasDeadDef) {3790          report(3791              "Instruction ending live segment on dead slot has no dead flag",3792              MI);3793          report_context(LR, VRegOrUnit, LaneMask);3794          report_context(S);3795        }3796      } else {3797        if (!hasRead) {3798          // When tracking subregister liveness, the main range must start new3799          // values on partial register writes, even if there is no read.3800          if (!MRI->shouldTrackSubRegLiveness(VRegOrUnit.asVirtualReg()) ||3801              LaneMask.any() || !hasSubRegDef) {3802            report("Instruction ending live segment doesn't read the register",3803                   MI);3804            report_context(LR, VRegOrUnit, LaneMask);3805            report_context(S);3806          }3807        }3808      }3809    }3810  }3811 3812  // Now check all the basic blocks in this live segment.3813  MachineFunction::const_iterator MFI = MBB->getIterator();3814  // Is this live segment the beginning of a non-PHIDef VN?3815  if (S.start == VNI->def && !VNI->isPHIDef()) {3816    // Not live-in to any blocks.3817    if (MBB == EndMBB)3818      return;3819    // Skip this block.3820    ++MFI;3821  }3822 3823  SmallVector<SlotIndex, 4> Undefs;3824  if (LaneMask.any()) {3825    LiveInterval &OwnerLI = LiveInts->getInterval(VRegOrUnit.asVirtualReg());3826    OwnerLI.computeSubRangeUndefs(Undefs, LaneMask, *MRI, *Indexes);3827  }3828 3829  while (true) {3830    assert(LiveInts->isLiveInToMBB(LR, &*MFI));3831    // We don't know how to track physregs into a landing pad.3832    if (!VRegOrUnit.isVirtualReg() && MFI->isEHPad()) {3833      if (&*MFI == EndMBB)3834        break;3835      ++MFI;3836      continue;3837    }3838 3839    // Is VNI a PHI-def in the current block?3840    bool IsPHI = VNI->isPHIDef() &&3841      VNI->def == LiveInts->getMBBStartIdx(&*MFI);3842 3843    // Check that VNI is live-out of all predecessors.3844    for (const MachineBasicBlock *Pred : MFI->predecessors()) {3845      SlotIndex PEnd = LiveInts->getMBBEndIdx(Pred);3846      // Predecessor of landing pad live-out on last call.3847      if (MFI->isEHPad()) {3848        for (const MachineInstr &MI : llvm::reverse(*Pred)) {3849          if (MI.isCall()) {3850            PEnd = Indexes->getInstructionIndex(MI).getBoundaryIndex();3851            break;3852          }3853        }3854      }3855      const VNInfo *PVNI = LR.getVNInfoBefore(PEnd);3856 3857      // All predecessors must have a live-out value. However for a phi3858      // instruction with subregister intervals3859      // only one of the subregisters (not necessarily the current one) needs to3860      // be defined.3861      if (!PVNI && (LaneMask.none() || !IsPHI)) {3862        if (LiveRangeCalc::isJointlyDominated(Pred, Undefs, *Indexes))3863          continue;3864        report("Register not marked live out of predecessor", Pred);3865        report_context(LR, VRegOrUnit, LaneMask);3866        report_context(*VNI);3867        OS << " live into " << printMBBReference(*MFI) << '@'3868           << LiveInts->getMBBStartIdx(&*MFI) << ", not live before " << PEnd3869           << '\n';3870        continue;3871      }3872 3873      // Only PHI-defs can take different predecessor values.3874      if (!IsPHI && PVNI != VNI) {3875        report("Different value live out of predecessor", Pred);3876        report_context(LR, VRegOrUnit, LaneMask);3877        OS << "Valno #" << PVNI->id << " live out of "3878           << printMBBReference(*Pred) << '@' << PEnd << "\nValno #" << VNI->id3879           << " live into " << printMBBReference(*MFI) << '@'3880           << LiveInts->getMBBStartIdx(&*MFI) << '\n';3881      }3882    }3883    if (&*MFI == EndMBB)3884      break;3885    ++MFI;3886  }3887}3888 3889void MachineVerifier::verifyLiveRange(const LiveRange &LR,3890                                      VirtRegOrUnit VRegOrUnit,3891                                      LaneBitmask LaneMask) {3892  for (const VNInfo *VNI : LR.valnos)3893    verifyLiveRangeValue(LR, VNI, VRegOrUnit, LaneMask);3894 3895  for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I)3896    verifyLiveRangeSegment(LR, I, VRegOrUnit, LaneMask);3897}3898 3899void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) {3900  Register Reg = LI.reg();3901  assert(Reg.isVirtual());3902  verifyLiveRange(LI, VirtRegOrUnit(Reg));3903 3904  if (LI.hasSubRanges()) {3905    LaneBitmask Mask;3906    LaneBitmask MaxMask = MRI->getMaxLaneMaskForVReg(Reg);3907    for (const LiveInterval::SubRange &SR : LI.subranges()) {3908      if ((Mask & SR.LaneMask).any()) {3909        report("Lane masks of sub ranges overlap in live interval", MF);3910        report_context(LI);3911      }3912      if ((SR.LaneMask & ~MaxMask).any()) {3913        report("Subrange lanemask is invalid", MF);3914        report_context(LI);3915      }3916      if (SR.empty()) {3917        report("Subrange must not be empty", MF);3918        report_context(SR, VirtRegOrUnit(LI.reg()), SR.LaneMask);3919      }3920      Mask |= SR.LaneMask;3921      verifyLiveRange(SR, VirtRegOrUnit(LI.reg()), SR.LaneMask);3922      if (!LI.covers(SR)) {3923        report("A Subrange is not covered by the main range", MF);3924        report_context(LI);3925      }3926    }3927  }3928 3929  // Check the LI only has one connected component.3930  ConnectedVNInfoEqClasses ConEQ(*LiveInts);3931  unsigned NumComp = ConEQ.Classify(LI);3932  if (NumComp > 1) {3933    report("Multiple connected components in live interval", MF);3934    report_context(LI);3935    for (unsigned comp = 0; comp != NumComp; ++comp) {3936      OS << comp << ": valnos";3937      for (const VNInfo *I : LI.valnos)3938        if (comp == ConEQ.getEqClass(I))3939          OS << ' ' << I->id;3940      OS << '\n';3941    }3942  }3943}3944 3945namespace {3946 3947  // FrameSetup and FrameDestroy can have zero adjustment, so using a single3948  // integer, we can't tell whether it is a FrameSetup or FrameDestroy if the3949  // value is zero.3950  // We use a bool plus an integer to capture the stack state.3951struct StackStateOfBB {3952  StackStateOfBB() = default;3953  StackStateOfBB(int EntryVal, int ExitVal, bool EntrySetup, bool ExitSetup)3954      : EntryValue(EntryVal), ExitValue(ExitVal), EntryIsSetup(EntrySetup),3955        ExitIsSetup(ExitSetup) {}3956 3957  // Can be negative, which means we are setting up a frame.3958  int EntryValue = 0;3959  int ExitValue = 0;3960  bool EntryIsSetup = false;3961  bool ExitIsSetup = false;3962};3963 3964} // end anonymous namespace3965 3966/// Make sure on every path through the CFG, a FrameSetup <n> is always followed3967/// by a FrameDestroy <n>, stack adjustments are identical on all3968/// CFG edges to a merge point, and frame is destroyed at end of a return block.3969void MachineVerifier::verifyStackFrame() {3970  unsigned FrameSetupOpcode   = TII->getCallFrameSetupOpcode();3971  unsigned FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();3972  if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)3973    return;3974 3975  SmallVector<StackStateOfBB, 8> SPState;3976  SPState.resize(MF->getNumBlockIDs());3977  df_iterator_default_set<const MachineBasicBlock*> Reachable;3978 3979  // Visit the MBBs in DFS order.3980  for (df_ext_iterator<const MachineFunction *,3981                       df_iterator_default_set<const MachineBasicBlock *>>3982       DFI = df_ext_begin(MF, Reachable), DFE = df_ext_end(MF, Reachable);3983       DFI != DFE; ++DFI) {3984    const MachineBasicBlock *MBB = *DFI;3985 3986    StackStateOfBB BBState;3987    // Check the exit state of the DFS stack predecessor.3988    if (DFI.getPathLength() >= 2) {3989      const MachineBasicBlock *StackPred = DFI.getPath(DFI.getPathLength() - 2);3990      assert(Reachable.count(StackPred) &&3991             "DFS stack predecessor is already visited.\n");3992      BBState.EntryValue = SPState[StackPred->getNumber()].ExitValue;3993      BBState.EntryIsSetup = SPState[StackPred->getNumber()].ExitIsSetup;3994      BBState.ExitValue = BBState.EntryValue;3995      BBState.ExitIsSetup = BBState.EntryIsSetup;3996    }3997 3998    if ((int)MBB->getCallFrameSize() != -BBState.EntryValue) {3999      report("Call frame size on entry does not match value computed from "4000             "predecessor",4001             MBB);4002      OS << "Call frame size on entry " << MBB->getCallFrameSize()4003         << " does not match value computed from predecessor "4004         << -BBState.EntryValue << '\n';4005    }4006 4007    // Update stack state by checking contents of MBB.4008    for (const auto &I : *MBB) {4009      if (I.getOpcode() == FrameSetupOpcode) {4010        if (BBState.ExitIsSetup)4011          report("FrameSetup is after another FrameSetup", &I);4012        if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())4013          report("AdjustsStack not set in presence of a frame pseudo "4014                 "instruction.", &I);4015        BBState.ExitValue -= TII->getFrameTotalSize(I);4016        BBState.ExitIsSetup = true;4017      }4018 4019      if (I.getOpcode() == FrameDestroyOpcode) {4020        int Size = TII->getFrameTotalSize(I);4021        if (!BBState.ExitIsSetup)4022          report("FrameDestroy is not after a FrameSetup", &I);4023        int AbsSPAdj = BBState.ExitValue < 0 ? -BBState.ExitValue :4024                                               BBState.ExitValue;4025        if (BBState.ExitIsSetup && AbsSPAdj != Size) {4026          report("FrameDestroy <n> is after FrameSetup <m>", &I);4027          OS << "FrameDestroy <" << Size << "> is after FrameSetup <"4028             << AbsSPAdj << ">.\n";4029        }4030        if (!MRI->isSSA() && !MF->getFrameInfo().adjustsStack())4031          report("AdjustsStack not set in presence of a frame pseudo "4032                 "instruction.", &I);4033        BBState.ExitValue += Size;4034        BBState.ExitIsSetup = false;4035      }4036    }4037    SPState[MBB->getNumber()] = BBState;4038 4039    // Make sure the exit state of any predecessor is consistent with the entry4040    // state.4041    for (const MachineBasicBlock *Pred : MBB->predecessors()) {4042      if (Reachable.count(Pred) &&4043          (SPState[Pred->getNumber()].ExitValue != BBState.EntryValue ||4044           SPState[Pred->getNumber()].ExitIsSetup != BBState.EntryIsSetup)) {4045        report("The exit stack state of a predecessor is inconsistent.", MBB);4046        OS << "Predecessor " << printMBBReference(*Pred) << " has exit state ("4047           << SPState[Pred->getNumber()].ExitValue << ", "4048           << SPState[Pred->getNumber()].ExitIsSetup << "), while "4049           << printMBBReference(*MBB) << " has entry state ("4050           << BBState.EntryValue << ", " << BBState.EntryIsSetup << ").\n";4051      }4052    }4053 4054    // Make sure the entry state of any successor is consistent with the exit4055    // state.4056    for (const MachineBasicBlock *Succ : MBB->successors()) {4057      if (Reachable.count(Succ) &&4058          (SPState[Succ->getNumber()].EntryValue != BBState.ExitValue ||4059           SPState[Succ->getNumber()].EntryIsSetup != BBState.ExitIsSetup)) {4060        report("The entry stack state of a successor is inconsistent.", MBB);4061        OS << "Successor " << printMBBReference(*Succ) << " has entry state ("4062           << SPState[Succ->getNumber()].EntryValue << ", "4063           << SPState[Succ->getNumber()].EntryIsSetup << "), while "4064           << printMBBReference(*MBB) << " has exit state ("4065           << BBState.ExitValue << ", " << BBState.ExitIsSetup << ").\n";4066      }4067    }4068 4069    // Make sure a basic block with return ends with zero stack adjustment.4070    if (!MBB->empty() && MBB->back().isReturn()) {4071      if (BBState.ExitIsSetup)4072        report("A return block ends with a FrameSetup.", MBB);4073      if (BBState.ExitValue)4074        report("A return block ends with a nonzero stack adjustment.", MBB);4075    }4076  }4077}4078 4079void MachineVerifier::verifyStackProtector() {4080  const MachineFrameInfo &MFI = MF->getFrameInfo();4081  if (!MFI.hasStackProtectorIndex())4082    return;4083  // Only applicable when the offsets of frame objects have been determined,4084  // which is indicated by a non-zero stack size.4085  if (!MFI.getStackSize())4086    return;4087  const TargetFrameLowering &TFI = *MF->getSubtarget().getFrameLowering();4088  bool StackGrowsDown =4089      TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;4090  unsigned FI = MFI.getStackProtectorIndex();4091  int64_t SPStart = MFI.getObjectOffset(FI);4092  int64_t SPEnd = SPStart + MFI.getObjectSize(FI);4093  for (unsigned I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) {4094    if (I == FI)4095      continue;4096    if (MFI.isDeadObjectIndex(I))4097      continue;4098    // FIXME: Skip non-default stack objects, as some targets may place them4099    // above the stack protector. This is a workaround for the fact that4100    // backends such as AArch64 may place SVE stack objects *above* the stack4101    // protector.4102    if (MFI.getStackID(I) != TargetStackID::Default)4103      continue;4104    // Skip variable-sized objects because they do not have a fixed offset.4105    if (MFI.isVariableSizedObjectIndex(I))4106      continue;4107    // FIXME: Skip spill slots which may be allocated above the stack protector.4108    // Ideally this would only skip callee-saved registers, but we don't have4109    // that information here. For example, spill-slots used for scavenging are4110    // not described in CalleeSavedInfo.4111    if (MFI.isSpillSlotObjectIndex(I))4112      continue;4113    int64_t ObjStart = MFI.getObjectOffset(I);4114    int64_t ObjEnd = ObjStart + MFI.getObjectSize(I);4115    if (SPStart < ObjEnd && ObjStart < SPEnd) {4116      report("Stack protector overlaps with another stack object", MF);4117      break;4118    }4119    if ((StackGrowsDown && SPStart <= ObjStart) ||4120        (!StackGrowsDown && SPStart >= ObjStart)) {4121      report("Stack protector is not the top-most object on the stack", MF);4122      break;4123    }4124  }4125}4126