brintos / llvm-project-archived public Read only

1//===-- RISCVInstrInfo.cpp - RISC-V Instruction Information -----*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains the RISC-V implementation of the TargetInstrInfo class.10//11//===----------------------------------------------------------------------===//12 13#include "RISCVInstrInfo.h"14#include "MCTargetDesc/RISCVBaseInfo.h"15#include "MCTargetDesc/RISCVMatInt.h"16#include "RISCV.h"17#include "RISCVMachineFunctionInfo.h"18#include "RISCVSubtarget.h"19#include "llvm/ADT/STLExtras.h"20#include "llvm/ADT/SmallVector.h"21#include "llvm/ADT/Statistic.h"22#include "llvm/Analysis/MemoryLocation.h"23#include "llvm/Analysis/ValueTracking.h"24#include "llvm/CodeGen/LiveIntervals.h"25#include "llvm/CodeGen/LiveVariables.h"26#include "llvm/CodeGen/MachineCombinerPattern.h"27#include "llvm/CodeGen/MachineInstrBuilder.h"28#include "llvm/CodeGen/MachineRegisterInfo.h"29#include "llvm/CodeGen/MachineTraceMetrics.h"30#include "llvm/CodeGen/RegisterScavenging.h"31#include "llvm/CodeGen/StackMaps.h"32#include "llvm/IR/DebugInfoMetadata.h"33#include "llvm/IR/Module.h"34#include "llvm/MC/MCDwarf.h"35#include "llvm/MC/MCInstBuilder.h"36#include "llvm/MC/TargetRegistry.h"37#include "llvm/Support/ErrorHandling.h"38 39using namespace llvm;40 41#define GEN_CHECK_COMPRESS_INSTR42#include "RISCVGenCompressInstEmitter.inc"43 44#define GET_INSTRINFO_CTOR_DTOR45#define GET_INSTRINFO_NAMED_OPS46#include "RISCVGenInstrInfo.inc"47 48#define DEBUG_TYPE "riscv-instr-info"49STATISTIC(NumVRegSpilled,50          "Number of registers within vector register groups spilled");51STATISTIC(NumVRegReloaded,52          "Number of registers within vector register groups reloaded");53 54static cl::opt<bool> PreferWholeRegisterMove(55    "riscv-prefer-whole-register-move", cl::init(false), cl::Hidden,56    cl::desc("Prefer whole register move for vector registers."));57 58static cl::opt<MachineTraceStrategy> ForceMachineCombinerStrategy(59    "riscv-force-machine-combiner-strategy", cl::Hidden,60    cl::desc("Force machine combiner to use a specific strategy for machine "61             "trace metrics evaluation."),62    cl::init(MachineTraceStrategy::TS_NumStrategies),63    cl::values(clEnumValN(MachineTraceStrategy::TS_Local, "local",64                          "Local strategy."),65               clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr",66                          "MinInstrCount strategy.")));67 68namespace llvm::RISCVVPseudosTable {69 70using namespace RISCV;71 72#define GET_RISCVVPseudosTable_IMPL73#include "RISCVGenSearchableTables.inc"74 75} // namespace llvm::RISCVVPseudosTable76 77namespace llvm::RISCV {78 79#define GET_RISCVMaskedPseudosTable_IMPL80#include "RISCVGenSearchableTables.inc"81 82} // end namespace llvm::RISCV83 84RISCVInstrInfo::RISCVInstrInfo(const RISCVSubtarget &STI)85    : RISCVGenInstrInfo(STI, RegInfo, RISCV::ADJCALLSTACKDOWN,86                        RISCV::ADJCALLSTACKUP),87      RegInfo(STI.getHwMode()), STI(STI) {}88 89#define GET_INSTRINFO_HELPERS90#include "RISCVGenInstrInfo.inc"91 92MCInst RISCVInstrInfo::getNop() const {93  if (STI.hasStdExtZca())94    return MCInstBuilder(RISCV::C_NOP);95  return MCInstBuilder(RISCV::ADDI)96      .addReg(RISCV::X0)97      .addReg(RISCV::X0)98      .addImm(0);99}100 101Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,102                                             int &FrameIndex) const {103  TypeSize Dummy = TypeSize::getZero();104  return isLoadFromStackSlot(MI, FrameIndex, Dummy);105}106 107static std::optional<unsigned> getLMULForRVVWholeLoadStore(unsigned Opcode) {108  switch (Opcode) {109  default:110    return std::nullopt;111  case RISCV::VS1R_V:112  case RISCV::VL1RE8_V:113  case RISCV::VL1RE16_V:114  case RISCV::VL1RE32_V:115  case RISCV::VL1RE64_V:116    return 1;117  case RISCV::VS2R_V:118  case RISCV::VL2RE8_V:119  case RISCV::VL2RE16_V:120  case RISCV::VL2RE32_V:121  case RISCV::VL2RE64_V:122    return 2;123  case RISCV::VS4R_V:124  case RISCV::VL4RE8_V:125  case RISCV::VL4RE16_V:126  case RISCV::VL4RE32_V:127  case RISCV::VL4RE64_V:128    return 4;129  case RISCV::VS8R_V:130  case RISCV::VL8RE8_V:131  case RISCV::VL8RE16_V:132  case RISCV::VL8RE32_V:133  case RISCV::VL8RE64_V:134    return 8;135  }136}137 138Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,139                                             int &FrameIndex,140                                             TypeSize &MemBytes) const {141  switch (MI.getOpcode()) {142  default:143    return 0;144  case RISCV::LB:145  case RISCV::LBU:146    MemBytes = TypeSize::getFixed(1);147    break;148  case RISCV::LH:149  case RISCV::LH_INX:150  case RISCV::LHU:151  case RISCV::FLH:152    MemBytes = TypeSize::getFixed(2);153    break;154  case RISCV::LW:155  case RISCV::LW_INX:156  case RISCV::FLW:157  case RISCV::LWU:158    MemBytes = TypeSize::getFixed(4);159    break;160  case RISCV::LD:161  case RISCV::LD_RV32:162  case RISCV::FLD:163    MemBytes = TypeSize::getFixed(8);164    break;165  case RISCV::VL1RE8_V:166  case RISCV::VL2RE8_V:167  case RISCV::VL4RE8_V:168  case RISCV::VL8RE8_V:169    if (!MI.getOperand(1).isFI())170      return Register();171    FrameIndex = MI.getOperand(1).getIndex();172    unsigned LMUL = *getLMULForRVVWholeLoadStore(MI.getOpcode());173    MemBytes = TypeSize::getScalable(RISCV::RVVBytesPerBlock * LMUL);174    return MI.getOperand(0).getReg();175  }176 177  if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&178      MI.getOperand(2).getImm() == 0) {179    FrameIndex = MI.getOperand(1).getIndex();180    return MI.getOperand(0).getReg();181  }182 183  return 0;184}185 186Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,187                                            int &FrameIndex) const {188  TypeSize Dummy = TypeSize::getZero();189  return isStoreToStackSlot(MI, FrameIndex, Dummy);190}191 192Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,193                                            int &FrameIndex,194                                            TypeSize &MemBytes) const {195  switch (MI.getOpcode()) {196  default:197    return 0;198  case RISCV::SB:199    MemBytes = TypeSize::getFixed(1);200    break;201  case RISCV::SH:202  case RISCV::SH_INX:203  case RISCV::FSH:204    MemBytes = TypeSize::getFixed(2);205    break;206  case RISCV::SW:207  case RISCV::SW_INX:208  case RISCV::FSW:209    MemBytes = TypeSize::getFixed(4);210    break;211  case RISCV::SD:212  case RISCV::SD_RV32:213  case RISCV::FSD:214    MemBytes = TypeSize::getFixed(8);215    break;216  case RISCV::VS1R_V:217  case RISCV::VS2R_V:218  case RISCV::VS4R_V:219  case RISCV::VS8R_V:220    if (!MI.getOperand(1).isFI())221      return Register();222    FrameIndex = MI.getOperand(1).getIndex();223    unsigned LMUL = *getLMULForRVVWholeLoadStore(MI.getOpcode());224    MemBytes = TypeSize::getScalable(RISCV::RVVBytesPerBlock * LMUL);225    return MI.getOperand(0).getReg();226  }227 228  if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&229      MI.getOperand(2).getImm() == 0) {230    FrameIndex = MI.getOperand(1).getIndex();231    return MI.getOperand(0).getReg();232  }233 234  return 0;235}236 237bool RISCVInstrInfo::isReMaterializableImpl(238    const MachineInstr &MI) const {239  switch (RISCV::getRVVMCOpcode(MI.getOpcode())) {240  case RISCV::VMV_V_X:241  case RISCV::VFMV_V_F:242  case RISCV::VMV_V_I:243  case RISCV::VMV_S_X:244  case RISCV::VFMV_S_F:245  case RISCV::VID_V:246    return MI.getOperand(1).isUndef();247  default:248    return TargetInstrInfo::isReMaterializableImpl(MI);249  }250}251 252static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,253                                        unsigned NumRegs) {254  return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;255}256 257static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI,258                                   const MachineBasicBlock &MBB,259                                   MachineBasicBlock::const_iterator MBBI,260                                   MachineBasicBlock::const_iterator &DefMBBI,261                                   RISCVVType::VLMUL LMul) {262  if (PreferWholeRegisterMove)263    return false;264 265  assert(MBBI->getOpcode() == TargetOpcode::COPY &&266         "Unexpected COPY instruction.");267  Register SrcReg = MBBI->getOperand(1).getReg();268  const TargetRegisterInfo *TRI = STI.getRegisterInfo();269 270  bool FoundDef = false;271  bool FirstVSetVLI = false;272  unsigned FirstSEW = 0;273  while (MBBI != MBB.begin()) {274    --MBBI;275    if (MBBI->isMetaInstruction())276      continue;277 278    if (RISCVInstrInfo::isVectorConfigInstr(*MBBI)) {279      // There is a vsetvli between COPY and source define instruction.280      // vy = def_vop ...  (producing instruction)281      // ...282      // vsetvli283      // ...284      // vx = COPY vy285      if (!FoundDef) {286        if (!FirstVSetVLI) {287          FirstVSetVLI = true;288          unsigned FirstVType = MBBI->getOperand(2).getImm();289          RISCVVType::VLMUL FirstLMul = RISCVVType::getVLMUL(FirstVType);290          FirstSEW = RISCVVType::getSEW(FirstVType);291          // The first encountered vsetvli must have the same lmul as the292          // register class of COPY.293          if (FirstLMul != LMul)294            return false;295        }296        // Only permit `vsetvli x0, x0, vtype` between COPY and the source297        // define instruction.298        if (!RISCVInstrInfo::isVLPreservingConfig(*MBBI))299          return false;300        continue;301      }302 303      // MBBI is the first vsetvli before the producing instruction.304      unsigned VType = MBBI->getOperand(2).getImm();305      // If there is a vsetvli between COPY and the producing instruction.306      if (FirstVSetVLI) {307        // If SEW is different, return false.308        if (RISCVVType::getSEW(VType) != FirstSEW)309          return false;310      }311 312      // If the vsetvli is tail undisturbed, keep the whole register move.313      if (!RISCVVType::isTailAgnostic(VType))314        return false;315 316      // The checking is conservative. We only have register classes for317      // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v318      // for fractional LMUL operations. However, we could not use the vsetvli319      // lmul for widening operations. The result of widening operation is320      // 2 x LMUL.321      return LMul == RISCVVType::getVLMUL(VType);322    } else if (MBBI->isInlineAsm() || MBBI->isCall()) {323      return false;324    } else if (MBBI->getNumDefs()) {325      // Check all the instructions which will change VL.326      // For example, vleff has implicit def VL.327      if (MBBI->modifiesRegister(RISCV::VL, /*TRI=*/nullptr))328        return false;329 330      // Only converting whole register copies to vmv.v.v when the defining331      // value appears in the explicit operands.332      for (const MachineOperand &MO : MBBI->explicit_operands()) {333        if (!MO.isReg() || !MO.isDef())334          continue;335        if (!FoundDef && TRI->regsOverlap(MO.getReg(), SrcReg)) {336          // We only permit the source of COPY has the same LMUL as the defined337          // operand.338          // There are cases we need to keep the whole register copy if the LMUL339          // is different.340          // For example,341          // $x0 = PseudoVSETIVLI 4, 73   // vsetivli zero, 4, e16,m2,ta,m342          // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2343          // # The COPY may be created by vlmul_trunc intrinsic.344          // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4345          //346          // After widening, the valid value will be 4 x e32 elements. If we347          // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.348          // FIXME: The COPY of subregister of Zvlsseg register will not be able349          // to convert to vmv.v.[v|i] under the constraint.350          if (MO.getReg() != SrcReg)351            return false;352 353          // In widening reduction instructions with LMUL_1 input vector case,354          // only checking the LMUL is insufficient due to reduction result is355          // always LMUL_1.356          // For example,357          // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu358          // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27359          // $v26 = COPY killed renamable $v8360          // After widening, The valid value will be 1 x e16 elements. If we361          // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.362          uint64_t TSFlags = MBBI->getDesc().TSFlags;363          if (RISCVII::isRVVWideningReduction(TSFlags))364            return false;365 366          // If the producing instruction does not depend on vsetvli, do not367          // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.368          if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))369            return false;370 371          // Found the definition.372          FoundDef = true;373          DefMBBI = MBBI;374          break;375        }376      }377    }378  }379 380  return false;381}382 383void RISCVInstrInfo::copyPhysRegVector(384    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,385    const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,386    const TargetRegisterClass *RegClass) const {387  const RISCVRegisterInfo *TRI = STI.getRegisterInfo();388  RISCVVType::VLMUL LMul = RISCVRI::getLMul(RegClass->TSFlags);389  unsigned NF = RISCVRI::getNF(RegClass->TSFlags);390 391  uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);392  uint16_t DstEncoding = TRI->getEncodingValue(DstReg);393  auto [LMulVal, Fractional] = RISCVVType::decodeVLMUL(LMul);394  assert(!Fractional && "It is impossible be fractional lmul here.");395  unsigned NumRegs = NF * LMulVal;396  bool ReversedCopy =397      forwardCopyWillClobberTuple(DstEncoding, SrcEncoding, NumRegs);398  if (ReversedCopy) {399    // If the src and dest overlap when copying a tuple, we need to copy the400    // registers in reverse.401    SrcEncoding += NumRegs - 1;402    DstEncoding += NumRegs - 1;403  }404 405  unsigned I = 0;406  auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)407      -> std::tuple<RISCVVType::VLMUL, const TargetRegisterClass &, unsigned,408                    unsigned, unsigned> {409    if (ReversedCopy) {410      // For reversed copying, if there are enough aligned registers(8/4/2), we411      // can do a larger copy(LMUL8/4/2).412      // Besides, we have already known that DstEncoding is larger than413      // SrcEncoding in forwardCopyWillClobberTuple, so the difference between414      // DstEncoding and SrcEncoding should be >= LMUL value we try to use to415      // avoid clobbering.416      uint16_t Diff = DstEncoding - SrcEncoding;417      if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&418          DstEncoding % 8 == 7)419        return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,420                RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};421      if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&422          DstEncoding % 4 == 3)423        return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,424                RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};425      if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&426          DstEncoding % 2 == 1)427        return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,428                RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};429      // Or we should do LMUL1 copying.430      return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,431              RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};432    }433 434    // For forward copying, if source register encoding and destination register435    // encoding are aligned to 8/4/2, we can do a LMUL8/4/2 copying.436    if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)437      return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,438              RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};439    if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)440      return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,441              RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};442    if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)443      return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,444              RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};445    // Or we should do LMUL1 copying.446    return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,447            RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};448  };449 450  while (I != NumRegs) {451    // For non-segment copying, we only do this once as the registers are always452    // aligned.453    // For segment copying, we may do this several times. If the registers are454    // aligned to larger LMUL, we can eliminate some copyings.455    auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =456        GetCopyInfo(SrcEncoding, DstEncoding);457    auto [NumCopied, _] = RISCVVType::decodeVLMUL(LMulCopied);458 459    MachineBasicBlock::const_iterator DefMBBI;460    if (LMul == LMulCopied &&461        isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {462      Opc = VVOpc;463      if (DefMBBI->getOpcode() == VIOpc)464        Opc = VIOpc;465    }466 467    // Emit actual copying.468    // For reversed copying, the encoding should be decreased.469    MCRegister ActualSrcReg = TRI->findVRegWithEncoding(470        RegClass, ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);471    MCRegister ActualDstReg = TRI->findVRegWithEncoding(472        RegClass, ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);473 474    auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), ActualDstReg);475    bool UseVMV_V_I = RISCV::getRVVMCOpcode(Opc) == RISCV::VMV_V_I;476    bool UseVMV = UseVMV_V_I || RISCV::getRVVMCOpcode(Opc) == RISCV::VMV_V_V;477    if (UseVMV)478      MIB.addReg(ActualDstReg, RegState::Undef);479    if (UseVMV_V_I)480      MIB = MIB.add(DefMBBI->getOperand(2));481    else482      MIB = MIB.addReg(ActualSrcReg, getKillRegState(KillSrc));483    if (UseVMV) {484      const MCInstrDesc &Desc = DefMBBI->getDesc();485      MIB.add(DefMBBI->getOperand(RISCVII::getVLOpNum(Desc)));  // AVL486      unsigned Log2SEW =487          DefMBBI->getOperand(RISCVII::getSEWOpNum(Desc)).getImm();488      MIB.addImm(Log2SEW ? Log2SEW : 3);                        // SEW489      MIB.addImm(0);                                            // tu, mu490      MIB.addReg(RISCV::VL, RegState::Implicit);491      MIB.addReg(RISCV::VTYPE, RegState::Implicit);492    }493    // Add an implicit read of the original source to silence the verifier494    // in the cases where some of the smaller VRs we're copying from might be495    // undef, caused by the fact that the original, larger source VR might not496    // be fully initialized at the time this COPY happens.497    MIB.addReg(SrcReg, RegState::Implicit);498 499    // If we are copying reversely, we should decrease the encoding.500    SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);501    DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);502    I += NumCopied;503  }504}505 506void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,507                                 MachineBasicBlock::iterator MBBI,508                                 const DebugLoc &DL, Register DstReg,509                                 Register SrcReg, bool KillSrc,510                                 bool RenamableDest, bool RenamableSrc) const {511  const TargetRegisterInfo *TRI = STI.getRegisterInfo();512  unsigned KillFlag = getKillRegState(KillSrc);513 514  if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {515    BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg)516        .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc))517        .addImm(0);518    return;519  }520 521  if (RISCV::GPRF16RegClass.contains(DstReg, SrcReg)) {522    BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR16INX), DstReg)523        .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));524    return;525  }526 527  if (RISCV::GPRF32RegClass.contains(DstReg, SrcReg)) {528    BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR32INX), DstReg)529        .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));530    return;531  }532 533  if (RISCV::GPRPairRegClass.contains(DstReg, SrcReg)) {534    if (STI.isRV32() && STI.hasStdExtZdinx()) {535      // On RV32_Zdinx, FMV.D will move a pair of registers to another pair of536      // registers, in one instruction.537      BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_D_IN32X), DstReg)538          .addReg(SrcReg, getRenamableRegState(RenamableSrc))539          .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));540      return;541    }542 543    MCRegister EvenReg = TRI->getSubReg(SrcReg, RISCV::sub_gpr_even);544    MCRegister OddReg = TRI->getSubReg(SrcReg, RISCV::sub_gpr_odd);545    // We need to correct the odd register of X0_Pair.546    if (OddReg == RISCV::DUMMY_REG_PAIR_WITH_X0)547      OddReg = RISCV::X0;548    assert(DstReg != RISCV::X0_Pair && "Cannot write to X0_Pair");549 550    // Emit an ADDI for both parts of GPRPair.551    BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),552            TRI->getSubReg(DstReg, RISCV::sub_gpr_even))553        .addReg(EvenReg, KillFlag)554        .addImm(0);555    BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),556            TRI->getSubReg(DstReg, RISCV::sub_gpr_odd))557        .addReg(OddReg, KillFlag)558        .addImm(0);559    return;560  }561 562  // Handle copy from csr563  if (RISCV::VCSRRegClass.contains(SrcReg) &&564      RISCV::GPRRegClass.contains(DstReg)) {565    BuildMI(MBB, MBBI, DL, get(RISCV::CSRRS), DstReg)566        .addImm(RISCVSysReg::lookupSysRegByName(TRI->getName(SrcReg))->Encoding)567        .addReg(RISCV::X0);568    return;569  }570 571  if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) {572    unsigned Opc;573    if (STI.hasStdExtZfh()) {574      Opc = RISCV::FSGNJ_H;575    } else {576      assert(STI.hasStdExtF() &&577             (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&578             "Unexpected extensions");579      // Zfhmin/Zfbfmin doesn't have FSGNJ_H, replace FSGNJ_H with FSGNJ_S.580      DstReg = TRI->getMatchingSuperReg(DstReg, RISCV::sub_16,581                                        &RISCV::FPR32RegClass);582      SrcReg = TRI->getMatchingSuperReg(SrcReg, RISCV::sub_16,583                                        &RISCV::FPR32RegClass);584      Opc = RISCV::FSGNJ_S;585    }586    BuildMI(MBB, MBBI, DL, get(Opc), DstReg)587        .addReg(SrcReg, KillFlag)588        .addReg(SrcReg, KillFlag);589    return;590  }591 592  if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {593    BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_S), DstReg)594        .addReg(SrcReg, KillFlag)595        .addReg(SrcReg, KillFlag);596    return;597  }598 599  if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {600    BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_D), DstReg)601        .addReg(SrcReg, KillFlag)602        .addReg(SrcReg, KillFlag);603    return;604  }605 606  if (RISCV::FPR32RegClass.contains(DstReg) &&607      RISCV::GPRRegClass.contains(SrcReg)) {608    BuildMI(MBB, MBBI, DL, get(RISCV::FMV_W_X), DstReg)609        .addReg(SrcReg, KillFlag);610    return;611  }612 613  if (RISCV::GPRRegClass.contains(DstReg) &&614      RISCV::FPR32RegClass.contains(SrcReg)) {615    BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_W), DstReg)616        .addReg(SrcReg, KillFlag);617    return;618  }619 620  if (RISCV::FPR64RegClass.contains(DstReg) &&621      RISCV::GPRRegClass.contains(SrcReg)) {622    assert(STI.getXLen() == 64 && "Unexpected GPR size");623    BuildMI(MBB, MBBI, DL, get(RISCV::FMV_D_X), DstReg)624        .addReg(SrcReg, KillFlag);625    return;626  }627 628  if (RISCV::GPRRegClass.contains(DstReg) &&629      RISCV::FPR64RegClass.contains(SrcReg)) {630    assert(STI.getXLen() == 64 && "Unexpected GPR size");631    BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_D), DstReg)632        .addReg(SrcReg, KillFlag);633    return;634  }635 636  // VR->VR copies.637  const TargetRegisterClass *RegClass =638      TRI->getCommonMinimalPhysRegClass(SrcReg, DstReg);639  if (RISCVRegisterInfo::isRVVRegClass(RegClass)) {640    copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RegClass);641    return;642  }643 644  llvm_unreachable("Impossible reg-to-reg copy");645}646 647void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,648                                         MachineBasicBlock::iterator I,649                                         Register SrcReg, bool IsKill, int FI,650                                         const TargetRegisterClass *RC,651                                         Register VReg,652                                         MachineInstr::MIFlag Flags) const {653  MachineFunction *MF = MBB.getParent();654  MachineFrameInfo &MFI = MF->getFrameInfo();655 656  unsigned Opcode;657  if (RISCV::GPRRegClass.hasSubClassEq(RC)) {658    Opcode = RegInfo.getRegSizeInBits(RISCV::GPRRegClass) == 32 ? RISCV::SW659                                                                : RISCV::SD;660  } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {661    Opcode = RISCV::SH_INX;662  } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {663    Opcode = RISCV::SW_INX;664  } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {665    Opcode = RISCV::PseudoRV32ZdinxSD;666  } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {667    Opcode = RISCV::FSH;668  } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {669    Opcode = RISCV::FSW;670  } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {671    Opcode = RISCV::FSD;672  } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {673    Opcode = RISCV::VS1R_V;674  } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {675    Opcode = RISCV::VS2R_V;676  } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {677    Opcode = RISCV::VS4R_V;678  } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {679    Opcode = RISCV::VS8R_V;680  } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))681    Opcode = RISCV::PseudoVSPILL2_M1;682  else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))683    Opcode = RISCV::PseudoVSPILL2_M2;684  else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))685    Opcode = RISCV::PseudoVSPILL2_M4;686  else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))687    Opcode = RISCV::PseudoVSPILL3_M1;688  else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))689    Opcode = RISCV::PseudoVSPILL3_M2;690  else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))691    Opcode = RISCV::PseudoVSPILL4_M1;692  else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))693    Opcode = RISCV::PseudoVSPILL4_M2;694  else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))695    Opcode = RISCV::PseudoVSPILL5_M1;696  else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))697    Opcode = RISCV::PseudoVSPILL6_M1;698  else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))699    Opcode = RISCV::PseudoVSPILL7_M1;700  else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))701    Opcode = RISCV::PseudoVSPILL8_M1;702  else703    llvm_unreachable("Can't store this register to stack slot");704 705  if (RISCVRegisterInfo::isRVVRegClass(RC)) {706    MachineMemOperand *MMO = MF->getMachineMemOperand(707        MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,708        TypeSize::getScalable(MFI.getObjectSize(FI)), MFI.getObjectAlign(FI));709 710    MFI.setStackID(FI, TargetStackID::ScalableVector);711    BuildMI(MBB, I, DebugLoc(), get(Opcode))712        .addReg(SrcReg, getKillRegState(IsKill))713        .addFrameIndex(FI)714        .addMemOperand(MMO)715        .setMIFlag(Flags);716    NumVRegSpilled += RegInfo.getRegSizeInBits(*RC) / RISCV::RVVBitsPerBlock;717  } else {718    MachineMemOperand *MMO = MF->getMachineMemOperand(719        MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOStore,720        MFI.getObjectSize(FI), MFI.getObjectAlign(FI));721 722    BuildMI(MBB, I, DebugLoc(), get(Opcode))723        .addReg(SrcReg, getKillRegState(IsKill))724        .addFrameIndex(FI)725        .addImm(0)726        .addMemOperand(MMO)727        .setMIFlag(Flags);728  }729}730 731void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,732                                          MachineBasicBlock::iterator I,733                                          Register DstReg, int FI,734                                          const TargetRegisterClass *RC,735                                          Register VReg,736                                          MachineInstr::MIFlag Flags) const {737  MachineFunction *MF = MBB.getParent();738  MachineFrameInfo &MFI = MF->getFrameInfo();739  DebugLoc DL =740      Flags & MachineInstr::FrameDestroy ? MBB.findDebugLoc(I) : DebugLoc();741 742  unsigned Opcode;743  if (RISCV::GPRRegClass.hasSubClassEq(RC)) {744    Opcode = RegInfo.getRegSizeInBits(RISCV::GPRRegClass) == 32 ? RISCV::LW745                                                                : RISCV::LD;746  } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {747    Opcode = RISCV::LH_INX;748  } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {749    Opcode = RISCV::LW_INX;750  } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {751    Opcode = RISCV::PseudoRV32ZdinxLD;752  } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {753    Opcode = RISCV::FLH;754  } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {755    Opcode = RISCV::FLW;756  } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {757    Opcode = RISCV::FLD;758  } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {759    Opcode = RISCV::VL1RE8_V;760  } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {761    Opcode = RISCV::VL2RE8_V;762  } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {763    Opcode = RISCV::VL4RE8_V;764  } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {765    Opcode = RISCV::VL8RE8_V;766  } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))767    Opcode = RISCV::PseudoVRELOAD2_M1;768  else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))769    Opcode = RISCV::PseudoVRELOAD2_M2;770  else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))771    Opcode = RISCV::PseudoVRELOAD2_M4;772  else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))773    Opcode = RISCV::PseudoVRELOAD3_M1;774  else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))775    Opcode = RISCV::PseudoVRELOAD3_M2;776  else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))777    Opcode = RISCV::PseudoVRELOAD4_M1;778  else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))779    Opcode = RISCV::PseudoVRELOAD4_M2;780  else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))781    Opcode = RISCV::PseudoVRELOAD5_M1;782  else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))783    Opcode = RISCV::PseudoVRELOAD6_M1;784  else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))785    Opcode = RISCV::PseudoVRELOAD7_M1;786  else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))787    Opcode = RISCV::PseudoVRELOAD8_M1;788  else789    llvm_unreachable("Can't load this register from stack slot");790 791  if (RISCVRegisterInfo::isRVVRegClass(RC)) {792    MachineMemOperand *MMO = MF->getMachineMemOperand(793        MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,794        TypeSize::getScalable(MFI.getObjectSize(FI)), MFI.getObjectAlign(FI));795 796    MFI.setStackID(FI, TargetStackID::ScalableVector);797    BuildMI(MBB, I, DL, get(Opcode), DstReg)798        .addFrameIndex(FI)799        .addMemOperand(MMO)800        .setMIFlag(Flags);801    NumVRegReloaded += RegInfo.getRegSizeInBits(*RC) / RISCV::RVVBitsPerBlock;802  } else {803    MachineMemOperand *MMO = MF->getMachineMemOperand(804        MachinePointerInfo::getFixedStack(*MF, FI), MachineMemOperand::MOLoad,805        MFI.getObjectSize(FI), MFI.getObjectAlign(FI));806 807    BuildMI(MBB, I, DL, get(Opcode), DstReg)808        .addFrameIndex(FI)809        .addImm(0)810        .addMemOperand(MMO)811        .setMIFlag(Flags);812  }813}814std::optional<unsigned> getFoldedOpcode(MachineFunction &MF, MachineInstr &MI,815                                        ArrayRef<unsigned> Ops,816                                        const RISCVSubtarget &ST) {817 818  // The below optimizations narrow the load so they are only valid for little819  // endian.820  // TODO: Support big endian by adding an offset into the frame object?821  if (MF.getDataLayout().isBigEndian())822    return std::nullopt;823 824  // Fold load from stack followed by sext.b/sext.h/sext.w/zext.b/zext.h/zext.w.825  if (Ops.size() != 1 || Ops[0] != 1)826    return std::nullopt;827 828  switch (MI.getOpcode()) {829  default:830    if (RISCVInstrInfo::isSEXT_W(MI))831      return RISCV::LW;832    if (RISCVInstrInfo::isZEXT_W(MI))833      return RISCV::LWU;834    if (RISCVInstrInfo::isZEXT_B(MI))835      return RISCV::LBU;836    break;837  case RISCV::SEXT_H:838    return RISCV::LH;839  case RISCV::SEXT_B:840    return RISCV::LB;841  case RISCV::ZEXT_H_RV32:842  case RISCV::ZEXT_H_RV64:843    return RISCV::LHU;844  }845 846  switch (RISCV::getRVVMCOpcode(MI.getOpcode())) {847  default:848    return std::nullopt;849  case RISCV::VMV_X_S: {850    unsigned Log2SEW =851        MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();852    if (ST.getXLen() < (1U << Log2SEW))853      return std::nullopt;854    switch (Log2SEW) {855    case 3:856      return RISCV::LB;857    case 4:858      return RISCV::LH;859    case 5:860      return RISCV::LW;861    case 6:862      return RISCV::LD;863    default:864      llvm_unreachable("Unexpected SEW");865    }866  }867  case RISCV::VFMV_F_S: {868    unsigned Log2SEW =869        MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();870    switch (Log2SEW) {871    case 4:872      return RISCV::FLH;873    case 5:874      return RISCV::FLW;875    case 6:876      return RISCV::FLD;877    default:878      llvm_unreachable("Unexpected SEW");879    }880  }881  }882}883 884// This is the version used during InlineSpiller::spillAroundUses885MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(886    MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,887    MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,888    VirtRegMap *VRM) const {889 890  std::optional<unsigned> LoadOpc = getFoldedOpcode(MF, MI, Ops, STI);891  if (!LoadOpc)892    return nullptr;893  Register DstReg = MI.getOperand(0).getReg();894  return BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(*LoadOpc),895                 DstReg)896      .addFrameIndex(FrameIndex)897      .addImm(0);898}899 900void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,901                            MachineBasicBlock::iterator MBBI,902                            const DebugLoc &DL, Register DstReg, uint64_t Val,903                            MachineInstr::MIFlag Flag, bool DstRenamable,904                            bool DstIsDead) const {905  Register SrcReg = RISCV::X0;906 907  // For RV32, allow a sign or unsigned 32 bit value.908  if (!STI.is64Bit() && !isInt<32>(Val)) {909    // If have a uimm32 it will still fit in a register so we can allow it.910    if (!isUInt<32>(Val))911      report_fatal_error("Should only materialize 32-bit constants for RV32");912 913    // Sign extend for generateInstSeq.914    Val = SignExtend64<32>(Val);915  }916 917  RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI);918  assert(!Seq.empty());919 920  bool SrcRenamable = false;921  unsigned Num = 0;922 923  for (const RISCVMatInt::Inst &Inst : Seq) {924    bool LastItem = ++Num == Seq.size();925    unsigned DstRegState = getDeadRegState(DstIsDead && LastItem) |926                           getRenamableRegState(DstRenamable);927    unsigned SrcRegState = getKillRegState(SrcReg != RISCV::X0) |928                           getRenamableRegState(SrcRenamable);929    switch (Inst.getOpndKind()) {930    case RISCVMatInt::Imm:931      BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))932          .addReg(DstReg, RegState::Define | DstRegState)933          .addImm(Inst.getImm())934          .setMIFlag(Flag);935      break;936    case RISCVMatInt::RegX0:937      BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))938          .addReg(DstReg, RegState::Define | DstRegState)939          .addReg(SrcReg, SrcRegState)940          .addReg(RISCV::X0)941          .setMIFlag(Flag);942      break;943    case RISCVMatInt::RegReg:944      BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))945          .addReg(DstReg, RegState::Define | DstRegState)946          .addReg(SrcReg, SrcRegState)947          .addReg(SrcReg, SrcRegState)948          .setMIFlag(Flag);949      break;950    case RISCVMatInt::RegImm:951      BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))952          .addReg(DstReg, RegState::Define | DstRegState)953          .addReg(SrcReg, SrcRegState)954          .addImm(Inst.getImm())955          .setMIFlag(Flag);956      break;957    }958 959    // Only the first instruction has X0 as its source.960    SrcReg = DstReg;961    SrcRenamable = DstRenamable;962  }963}964 965RISCVCC::CondCode RISCVInstrInfo::getCondFromBranchOpc(unsigned Opc) {966  switch (Opc) {967  default:968    return RISCVCC::COND_INVALID;969  case RISCV::BEQ:970  case RISCV::BEQI:971  case RISCV::CV_BEQIMM:972  case RISCV::QC_BEQI:973  case RISCV::QC_E_BEQI:974  case RISCV::NDS_BBC:975  case RISCV::NDS_BEQC:976    return RISCVCC::COND_EQ;977  case RISCV::BNE:978  case RISCV::BNEI:979  case RISCV::QC_BNEI:980  case RISCV::QC_E_BNEI:981  case RISCV::CV_BNEIMM:982  case RISCV::NDS_BBS:983  case RISCV::NDS_BNEC:984    return RISCVCC::COND_NE;985  case RISCV::BLT:986  case RISCV::QC_BLTI:987  case RISCV::QC_E_BLTI:988    return RISCVCC::COND_LT;989  case RISCV::BGE:990  case RISCV::QC_BGEI:991  case RISCV::QC_E_BGEI:992    return RISCVCC::COND_GE;993  case RISCV::BLTU:994  case RISCV::QC_BLTUI:995  case RISCV::QC_E_BLTUI:996    return RISCVCC::COND_LTU;997  case RISCV::BGEU:998  case RISCV::QC_BGEUI:999  case RISCV::QC_E_BGEUI:1000    return RISCVCC::COND_GEU;1001  }1002}1003 1004bool RISCVInstrInfo::evaluateCondBranch(RISCVCC::CondCode CC, int64_t C0,1005                                        int64_t C1) {1006  switch (CC) {1007  default:1008    llvm_unreachable("Unexpected CC");1009  case RISCVCC::COND_EQ:1010    return C0 == C1;1011  case RISCVCC::COND_NE:1012    return C0 != C1;1013  case RISCVCC::COND_LT:1014    return C0 < C1;1015  case RISCVCC::COND_GE:1016    return C0 >= C1;1017  case RISCVCC::COND_LTU:1018    return (uint64_t)C0 < (uint64_t)C1;1019  case RISCVCC::COND_GEU:1020    return (uint64_t)C0 >= (uint64_t)C1;1021  }1022}1023 1024// The contents of values added to Cond are not examined outside of1025// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we1026// push BranchOpcode, Reg1, Reg2.1027static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,1028                            SmallVectorImpl<MachineOperand> &Cond) {1029  // Block ends with fall-through condbranch.1030  assert(LastInst.getDesc().isConditionalBranch() &&1031         "Unknown conditional branch");1032  Target = LastInst.getOperand(2).getMBB();1033  Cond.push_back(MachineOperand::CreateImm(LastInst.getOpcode()));1034  Cond.push_back(LastInst.getOperand(0));1035  Cond.push_back(LastInst.getOperand(1));1036}1037 1038static unsigned getInverseXqcicmOpcode(unsigned Opcode) {1039  switch (Opcode) {1040  default:1041    llvm_unreachable("Unexpected Opcode");1042  case RISCV::QC_MVEQ:1043    return RISCV::QC_MVNE;1044  case RISCV::QC_MVNE:1045    return RISCV::QC_MVEQ;1046  case RISCV::QC_MVLT:1047    return RISCV::QC_MVGE;1048  case RISCV::QC_MVGE:1049    return RISCV::QC_MVLT;1050  case RISCV::QC_MVLTU:1051    return RISCV::QC_MVGEU;1052  case RISCV::QC_MVGEU:1053    return RISCV::QC_MVLTU;1054  case RISCV::QC_MVEQI:1055    return RISCV::QC_MVNEI;1056  case RISCV::QC_MVNEI:1057    return RISCV::QC_MVEQI;1058  case RISCV::QC_MVLTI:1059    return RISCV::QC_MVGEI;1060  case RISCV::QC_MVGEI:1061    return RISCV::QC_MVLTI;1062  case RISCV::QC_MVLTUI:1063    return RISCV::QC_MVGEUI;1064  case RISCV::QC_MVGEUI:1065    return RISCV::QC_MVLTUI;1066  }1067}1068 1069unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, unsigned SelectOpc) {1070  switch (SelectOpc) {1071  default:1072    switch (CC) {1073    default:1074      llvm_unreachable("Unexpected condition code!");1075    case RISCVCC::COND_EQ:1076      return RISCV::BEQ;1077    case RISCVCC::COND_NE:1078      return RISCV::BNE;1079    case RISCVCC::COND_LT:1080      return RISCV::BLT;1081    case RISCVCC::COND_GE:1082      return RISCV::BGE;1083    case RISCVCC::COND_LTU:1084      return RISCV::BLTU;1085    case RISCVCC::COND_GEU:1086      return RISCV::BGEU;1087    }1088    break;1089  case RISCV::Select_GPR_Using_CC_Imm5_Zibi:1090    switch (CC) {1091    default:1092      llvm_unreachable("Unexpected condition code!");1093    case RISCVCC::COND_EQ:1094      return RISCV::BEQI;1095    case RISCVCC::COND_NE:1096      return RISCV::BNEI;1097    }1098    break;1099  case RISCV::Select_GPR_Using_CC_SImm5_CV:1100    switch (CC) {1101    default:1102      llvm_unreachable("Unexpected condition code!");1103    case RISCVCC::COND_EQ:1104      return RISCV::CV_BEQIMM;1105    case RISCVCC::COND_NE:1106      return RISCV::CV_BNEIMM;1107    }1108    break;1109  case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:1110    switch (CC) {1111    default:1112      llvm_unreachable("Unexpected condition code!");1113    case RISCVCC::COND_EQ:1114      return RISCV::QC_BEQI;1115    case RISCVCC::COND_NE:1116      return RISCV::QC_BNEI;1117    case RISCVCC::COND_LT:1118      return RISCV::QC_BLTI;1119    case RISCVCC::COND_GE:1120      return RISCV::QC_BGEI;1121    }1122    break;1123  case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:1124    switch (CC) {1125    default:1126      llvm_unreachable("Unexpected condition code!");1127    case RISCVCC::COND_LTU:1128      return RISCV::QC_BLTUI;1129    case RISCVCC::COND_GEU:1130      return RISCV::QC_BGEUI;1131    }1132    break;1133  case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:1134    switch (CC) {1135    default:1136      llvm_unreachable("Unexpected condition code!");1137    case RISCVCC::COND_EQ:1138      return RISCV::QC_E_BEQI;1139    case RISCVCC::COND_NE:1140      return RISCV::QC_E_BNEI;1141    case RISCVCC::COND_LT:1142      return RISCV::QC_E_BLTI;1143    case RISCVCC::COND_GE:1144      return RISCV::QC_E_BGEI;1145    }1146    break;1147  case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:1148    switch (CC) {1149    default:1150      llvm_unreachable("Unexpected condition code!");1151    case RISCVCC::COND_LTU:1152      return RISCV::QC_E_BLTUI;1153    case RISCVCC::COND_GEU:1154      return RISCV::QC_E_BGEUI;1155    }1156    break;1157  case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:1158    switch (CC) {1159    default:1160      llvm_unreachable("Unexpected condition code!");1161    case RISCVCC::COND_EQ:1162      return RISCV::NDS_BBC;1163    case RISCVCC::COND_NE:1164      return RISCV::NDS_BBS;1165    }1166    break;1167  case RISCV::Select_GPR_Using_CC_UImm7_NDS:1168    switch (CC) {1169    default:1170      llvm_unreachable("Unexpected condition code!");1171    case RISCVCC::COND_EQ:1172      return RISCV::NDS_BEQC;1173    case RISCVCC::COND_NE:1174      return RISCV::NDS_BNEC;1175    }1176    break;1177  }1178}1179 1180RISCVCC::CondCode RISCVCC::getInverseBranchCondition(RISCVCC::CondCode CC) {1181  switch (CC) {1182  default:1183    llvm_unreachable("Unrecognized conditional branch");1184  case RISCVCC::COND_EQ:1185    return RISCVCC::COND_NE;1186  case RISCVCC::COND_NE:1187    return RISCVCC::COND_EQ;1188  case RISCVCC::COND_LT:1189    return RISCVCC::COND_GE;1190  case RISCVCC::COND_GE:1191    return RISCVCC::COND_LT;1192  case RISCVCC::COND_LTU:1193    return RISCVCC::COND_GEU;1194  case RISCVCC::COND_GEU:1195    return RISCVCC::COND_LTU;1196  }1197}1198 1199bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB,1200                                   MachineBasicBlock *&TBB,1201                                   MachineBasicBlock *&FBB,1202                                   SmallVectorImpl<MachineOperand> &Cond,1203                                   bool AllowModify) const {1204  TBB = FBB = nullptr;1205  Cond.clear();1206 1207  // If the block has no terminators, it just falls into the block after it.1208  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();1209  if (I == MBB.end() || !isUnpredicatedTerminator(*I))1210    return false;1211 1212  // Count the number of terminators and find the first unconditional or1213  // indirect branch.1214  MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();1215  int NumTerminators = 0;1216  for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J);1217       J++) {1218    NumTerminators++;1219    if (J->getDesc().isUnconditionalBranch() ||1220        J->getDesc().isIndirectBranch()) {1221      FirstUncondOrIndirectBr = J.getReverse();1222    }1223  }1224 1225  // If AllowModify is true, we can erase any terminators after1226  // FirstUncondOrIndirectBR.1227  if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {1228    while (std::next(FirstUncondOrIndirectBr) != MBB.end()) {1229      std::next(FirstUncondOrIndirectBr)->eraseFromParent();1230      NumTerminators--;1231    }1232    I = FirstUncondOrIndirectBr;1233  }1234 1235  // We can't handle blocks that end in an indirect branch.1236  if (I->getDesc().isIndirectBranch())1237    return true;1238 1239  // We can't handle Generic branch opcodes from Global ISel.1240  if (I->isPreISelOpcode())1241    return true;1242 1243  // We can't handle blocks with more than 2 terminators.1244  if (NumTerminators > 2)1245    return true;1246 1247  // Handle a single unconditional branch.1248  if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {1249    TBB = getBranchDestBlock(*I);1250    return false;1251  }1252 1253  // Handle a single conditional branch.1254  if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {1255    parseCondBranch(*I, TBB, Cond);1256    return false;1257  }1258 1259  // Handle a conditional branch followed by an unconditional branch.1260  if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() &&1261      I->getDesc().isUnconditionalBranch()) {1262    parseCondBranch(*std::prev(I), TBB, Cond);1263    FBB = getBranchDestBlock(*I);1264    return false;1265  }1266 1267  // Otherwise, we can't handle this.1268  return true;1269}1270 1271unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB,1272                                      int *BytesRemoved) const {1273  if (BytesRemoved)1274    *BytesRemoved = 0;1275  MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();1276  if (I == MBB.end())1277    return 0;1278 1279  if (!I->getDesc().isUnconditionalBranch() &&1280      !I->getDesc().isConditionalBranch())1281    return 0;1282 1283  // Remove the branch.1284  if (BytesRemoved)1285    *BytesRemoved += getInstSizeInBytes(*I);1286  I->eraseFromParent();1287 1288  I = MBB.end();1289 1290  if (I == MBB.begin())1291    return 1;1292  --I;1293  if (!I->getDesc().isConditionalBranch())1294    return 1;1295 1296  // Remove the branch.1297  if (BytesRemoved)1298    *BytesRemoved += getInstSizeInBytes(*I);1299  I->eraseFromParent();1300  return 2;1301}1302 1303// Inserts a branch into the end of the specific MachineBasicBlock, returning1304// the number of instructions inserted.1305unsigned RISCVInstrInfo::insertBranch(1306    MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,1307    ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {1308  if (BytesAdded)1309    *BytesAdded = 0;1310 1311  // Shouldn't be a fall through.1312  assert(TBB && "insertBranch must not be told to insert a fallthrough");1313  assert((Cond.size() == 3 || Cond.size() == 0) &&1314         "RISC-V branch conditions have two components!");1315 1316  // Unconditional branch.1317  if (Cond.empty()) {1318    MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB);1319    if (BytesAdded)1320      *BytesAdded += getInstSizeInBytes(MI);1321    return 1;1322  }1323 1324  // Either a one or two-way conditional branch.1325  MachineInstr &CondMI = *BuildMI(&MBB, DL, get(Cond[0].getImm()))1326                              .add(Cond[1])1327                              .add(Cond[2])1328                              .addMBB(TBB);1329  if (BytesAdded)1330    *BytesAdded += getInstSizeInBytes(CondMI);1331 1332  // One-way conditional branch.1333  if (!FBB)1334    return 1;1335 1336  // Two-way conditional branch.1337  MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB);1338  if (BytesAdded)1339    *BytesAdded += getInstSizeInBytes(MI);1340  return 2;1341}1342 1343void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,1344                                          MachineBasicBlock &DestBB,1345                                          MachineBasicBlock &RestoreBB,1346                                          const DebugLoc &DL, int64_t BrOffset,1347                                          RegScavenger *RS) const {1348  assert(RS && "RegScavenger required for long branching");1349  assert(MBB.empty() &&1350         "new block should be inserted for expanding unconditional branch");1351  assert(MBB.pred_size() == 1);1352  assert(RestoreBB.empty() &&1353         "restore block should be inserted for restoring clobbered registers");1354 1355  MachineFunction *MF = MBB.getParent();1356  MachineRegisterInfo &MRI = MF->getRegInfo();1357  RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();1358  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();1359 1360  if (!isInt<32>(BrOffset))1361    report_fatal_error(1362        "Branch offsets outside of the signed 32-bit range not supported");1363 1364  // FIXME: A virtual register must be used initially, as the register1365  // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch1366  // uses the same workaround).1367  Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRJALRRegClass);1368  auto II = MBB.end();1369  // We may also update the jump target to RestoreBB later.1370  MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump))1371                          .addReg(ScratchReg, RegState::Define | RegState::Dead)1372                          .addMBB(&DestBB, RISCVII::MO_CALL);1373 1374  RS->enterBasicBlockEnd(MBB);1375  Register TmpGPR =1376      RS->scavengeRegisterBackwards(RISCV::GPRRegClass, MI.getIterator(),1377                                    /*RestoreAfter=*/false, /*SpAdj=*/0,1378                                    /*AllowSpill=*/false);1379  if (TmpGPR.isValid())1380    RS->setRegUsed(TmpGPR);1381  else {1382    // The case when there is no scavenged register needs special handling.1383 1384    // Pick s11(or s1 for rve) because it doesn't make a difference.1385    TmpGPR = STI.hasStdExtE() ? RISCV::X9 : RISCV::X27;1386 1387    int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();1388    if (FrameIndex == -1)1389      report_fatal_error("underestimated function size");1390 1391    storeRegToStackSlot(MBB, MI, TmpGPR, /*IsKill=*/true, FrameIndex,1392                        &RISCV::GPRRegClass, Register());1393    TRI->eliminateFrameIndex(std::prev(MI.getIterator()),1394                             /*SpAdj=*/0, /*FIOperandNum=*/1);1395 1396    MI.getOperand(1).setMBB(&RestoreBB);1397 1398    loadRegFromStackSlot(RestoreBB, RestoreBB.end(), TmpGPR, FrameIndex,1399                         &RISCV::GPRRegClass, Register());1400    TRI->eliminateFrameIndex(RestoreBB.back(),1401                             /*SpAdj=*/0, /*FIOperandNum=*/1);1402  }1403 1404  MRI.replaceRegWith(ScratchReg, TmpGPR);1405  MRI.clearVirtRegs();1406}1407 1408bool RISCVInstrInfo::reverseBranchCondition(1409    SmallVectorImpl<MachineOperand> &Cond) const {1410  assert((Cond.size() == 3) && "Invalid branch condition!");1411  switch (Cond[0].getImm()) {1412  default:1413    llvm_unreachable("Unknown conditional branch!");1414  case RISCV::BEQ:1415    Cond[0].setImm(RISCV::BNE);1416    break;1417  case RISCV::BEQI:1418    Cond[0].setImm(RISCV::BNEI);1419    break;1420  case RISCV::BNE:1421    Cond[0].setImm(RISCV::BEQ);1422    break;1423  case RISCV::BNEI:1424    Cond[0].setImm(RISCV::BEQI);1425    break;1426  case RISCV::BLT:1427    Cond[0].setImm(RISCV::BGE);1428    break;1429  case RISCV::BGE:1430    Cond[0].setImm(RISCV::BLT);1431    break;1432  case RISCV::BLTU:1433    Cond[0].setImm(RISCV::BGEU);1434    break;1435  case RISCV::BGEU:1436    Cond[0].setImm(RISCV::BLTU);1437    break;1438  case RISCV::CV_BEQIMM:1439    Cond[0].setImm(RISCV::CV_BNEIMM);1440    break;1441  case RISCV::CV_BNEIMM:1442    Cond[0].setImm(RISCV::CV_BEQIMM);1443    break;1444  case RISCV::QC_BEQI:1445    Cond[0].setImm(RISCV::QC_BNEI);1446    break;1447  case RISCV::QC_BNEI:1448    Cond[0].setImm(RISCV::QC_BEQI);1449    break;1450  case RISCV::QC_BGEI:1451    Cond[0].setImm(RISCV::QC_BLTI);1452    break;1453  case RISCV::QC_BLTI:1454    Cond[0].setImm(RISCV::QC_BGEI);1455    break;1456  case RISCV::QC_BGEUI:1457    Cond[0].setImm(RISCV::QC_BLTUI);1458    break;1459  case RISCV::QC_BLTUI:1460    Cond[0].setImm(RISCV::QC_BGEUI);1461    break;1462  case RISCV::QC_E_BEQI:1463    Cond[0].setImm(RISCV::QC_E_BNEI);1464    break;1465  case RISCV::QC_E_BNEI:1466    Cond[0].setImm(RISCV::QC_E_BEQI);1467    break;1468  case RISCV::QC_E_BGEI:1469    Cond[0].setImm(RISCV::QC_E_BLTI);1470    break;1471  case RISCV::QC_E_BLTI:1472    Cond[0].setImm(RISCV::QC_E_BGEI);1473    break;1474  case RISCV::QC_E_BGEUI:1475    Cond[0].setImm(RISCV::QC_E_BLTUI);1476    break;1477  case RISCV::QC_E_BLTUI:1478    Cond[0].setImm(RISCV::QC_E_BGEUI);1479    break;1480  case RISCV::NDS_BBC:1481    Cond[0].setImm(RISCV::NDS_BBS);1482    break;1483  case RISCV::NDS_BBS:1484    Cond[0].setImm(RISCV::NDS_BBC);1485    break;1486  case RISCV::NDS_BEQC:1487    Cond[0].setImm(RISCV::NDS_BNEC);1488    break;1489  case RISCV::NDS_BNEC:1490    Cond[0].setImm(RISCV::NDS_BEQC);1491    break;1492  }1493 1494  return false;1495}1496 1497// Return true if the instruction is a load immediate instruction (i.e.1498// ADDI x0, imm).1499static bool isLoadImm(const MachineInstr *MI, int64_t &Imm) {1500  if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&1501      MI->getOperand(1).getReg() == RISCV::X0) {1502    Imm = MI->getOperand(2).getImm();1503    return true;1504  }1505  return false;1506}1507 1508bool RISCVInstrInfo::isFromLoadImm(const MachineRegisterInfo &MRI,1509                                   const MachineOperand &Op, int64_t &Imm) {1510  // Either a load from immediate instruction or X0.1511  if (!Op.isReg())1512    return false;1513 1514  Register Reg = Op.getReg();1515  if (Reg == RISCV::X0) {1516    Imm = 0;1517    return true;1518  }1519  return Reg.isVirtual() && isLoadImm(MRI.getVRegDef(Reg), Imm);1520}1521 1522bool RISCVInstrInfo::optimizeCondBranch(MachineInstr &MI) const {1523  bool IsSigned = false;1524  bool IsEquality = false;1525  switch (MI.getOpcode()) {1526  default:1527    return false;1528  case RISCV::BEQ:1529  case RISCV::BNE:1530    IsEquality = true;1531    break;1532  case RISCV::BGE:1533  case RISCV::BLT:1534    IsSigned = true;1535    break;1536  case RISCV::BGEU:1537  case RISCV::BLTU:1538    break;1539  }1540 1541  MachineBasicBlock *MBB = MI.getParent();1542  MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();1543 1544  const MachineOperand &LHS = MI.getOperand(0);1545  const MachineOperand &RHS = MI.getOperand(1);1546  MachineBasicBlock *TBB = MI.getOperand(2).getMBB();1547 1548  RISCVCC::CondCode CC = getCondFromBranchOpc(MI.getOpcode());1549  assert(CC != RISCVCC::COND_INVALID);1550 1551  // Canonicalize conditional branches which can be constant folded into1552  // beqz or bnez.  We can't modify the CFG here.1553  int64_t C0, C1;1554  if (isFromLoadImm(MRI, LHS, C0) && isFromLoadImm(MRI, RHS, C1)) {1555    unsigned NewOpc = evaluateCondBranch(CC, C0, C1) ? RISCV::BEQ : RISCV::BNE;1556    // Build the new branch and remove the old one.1557    BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))1558        .addReg(RISCV::X0)1559        .addReg(RISCV::X0)1560        .addMBB(TBB);1561    MI.eraseFromParent();1562    return true;1563  }1564 1565  if (IsEquality)1566    return false;1567 1568  // For two constants C0 and C1 from1569  // ```1570  // li Y, C01571  // li Z, C11572  // ```1573  // 1. if C1 = C0 + 11574  // we can turn:1575  //  (a) blt Y, X -> bge X, Z1576  //  (b) bge Y, X -> blt X, Z1577  //1578  // 2. if C1 = C0 - 11579  // we can turn:1580  //  (a) blt X, Y -> bge Z, X1581  //  (b) bge X, Y -> blt Z, X1582  //1583  // To make sure this optimization is really beneficial, we only1584  // optimize for cases where Y had only one use (i.e. only used by the branch).1585  // Try to find the register for constant Z; return1586  // invalid register otherwise.1587  auto searchConst = [&](int64_t C1) -> Register {1588    MachineBasicBlock::reverse_iterator II(&MI), E = MBB->rend();1589    auto DefC1 = std::find_if(++II, E, [&](const MachineInstr &I) -> bool {1590      int64_t Imm;1591      return isLoadImm(&I, Imm) && Imm == C1 &&1592             I.getOperand(0).getReg().isVirtual();1593    });1594    if (DefC1 != E)1595      return DefC1->getOperand(0).getReg();1596 1597    return Register();1598  };1599 1600  unsigned NewOpc = RISCVCC::getBrCond(getInverseBranchCondition(CC));1601 1602  // Might be case 1.1603  // Don't change 0 to 1 since we can use x0.1604  // For unsigned cases changing -1U to 0 would be incorrect.1605  // The incorrect case for signed would be INT_MAX, but isFromLoadImm can't1606  // return that.1607  if (isFromLoadImm(MRI, LHS, C0) && C0 != 0 && LHS.getReg().isVirtual() &&1608      MRI.hasOneUse(LHS.getReg()) && (IsSigned || C0 != -1)) {1609    assert(isInt<12>(C0) && "Unexpected immediate");1610    if (Register RegZ = searchConst(C0 + 1)) {1611      BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))1612          .add(RHS)1613          .addReg(RegZ)1614          .addMBB(TBB);1615      // We might extend the live range of Z, clear its kill flag to1616      // account for this.1617      MRI.clearKillFlags(RegZ);1618      MI.eraseFromParent();1619      return true;1620    }1621  }1622 1623  // Might be case 2.1624  // For signed cases we don't want to change 0 since we can use x0.1625  // For unsigned cases changing 0 to -1U would be incorrect.1626  // The incorrect case for signed would be INT_MIN, but isFromLoadImm can't1627  // return that.1628  if (isFromLoadImm(MRI, RHS, C0) && C0 != 0 && RHS.getReg().isVirtual() &&1629      MRI.hasOneUse(RHS.getReg())) {1630    assert(isInt<12>(C0) && "Unexpected immediate");1631    if (Register RegZ = searchConst(C0 - 1)) {1632      BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))1633          .addReg(RegZ)1634          .add(LHS)1635          .addMBB(TBB);1636      // We might extend the live range of Z, clear its kill flag to1637      // account for this.1638      MRI.clearKillFlags(RegZ);1639      MI.eraseFromParent();1640      return true;1641    }1642  }1643 1644  return false;1645}1646 1647MachineBasicBlock *1648RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {1649  assert(MI.getDesc().isBranch() && "Unexpected opcode!");1650  // The branch target is always the last operand.1651  int NumOp = MI.getNumExplicitOperands();1652  return MI.getOperand(NumOp - 1).getMBB();1653}1654 1655bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,1656                                           int64_t BrOffset) const {1657  unsigned XLen = STI.getXLen();1658  // Ideally we could determine the supported branch offset from the1659  // RISCVII::FormMask, but this can't be used for Pseudo instructions like1660  // PseudoBR.1661  switch (BranchOp) {1662  default:1663    llvm_unreachable("Unexpected opcode!");1664  case RISCV::NDS_BBC:1665  case RISCV::NDS_BBS:1666  case RISCV::NDS_BEQC:1667  case RISCV::NDS_BNEC:1668    return isInt<11>(BrOffset);1669  case RISCV::BEQ:1670  case RISCV::BNE:1671  case RISCV::BLT:1672  case RISCV::BGE:1673  case RISCV::BLTU:1674  case RISCV::BGEU:1675  case RISCV::BEQI:1676  case RISCV::BNEI:1677  case RISCV::CV_BEQIMM:1678  case RISCV::CV_BNEIMM:1679  case RISCV::QC_BEQI:1680  case RISCV::QC_BNEI:1681  case RISCV::QC_BGEI:1682  case RISCV::QC_BLTI:1683  case RISCV::QC_BLTUI:1684  case RISCV::QC_BGEUI:1685  case RISCV::QC_E_BEQI:1686  case RISCV::QC_E_BNEI:1687  case RISCV::QC_E_BGEI:1688  case RISCV::QC_E_BLTI:1689  case RISCV::QC_E_BLTUI:1690  case RISCV::QC_E_BGEUI:1691    return isInt<13>(BrOffset);1692  case RISCV::JAL:1693  case RISCV::PseudoBR:1694    return isInt<21>(BrOffset);1695  case RISCV::PseudoJump:1696    return isInt<32>(SignExtend64(BrOffset + 0x800, XLen));1697  }1698}1699 1700// If the operation has a predicated pseudo instruction, return the pseudo1701// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.1702// TODO: Support more operations.1703unsigned getPredicatedOpcode(unsigned Opcode) {1704  // clang-format off1705  switch (Opcode) {1706  case RISCV::ADD:   return RISCV::PseudoCCADD;1707  case RISCV::SUB:   return RISCV::PseudoCCSUB;1708  case RISCV::SLL:   return RISCV::PseudoCCSLL;1709  case RISCV::SRL:   return RISCV::PseudoCCSRL;1710  case RISCV::SRA:   return RISCV::PseudoCCSRA;1711  case RISCV::AND:   return RISCV::PseudoCCAND;1712  case RISCV::OR:    return RISCV::PseudoCCOR;1713  case RISCV::XOR:   return RISCV::PseudoCCXOR;1714  case RISCV::MAX:   return RISCV::PseudoCCMAX;1715  case RISCV::MAXU:  return RISCV::PseudoCCMAXU;1716  case RISCV::MIN:   return RISCV::PseudoCCMIN;1717  case RISCV::MINU:  return RISCV::PseudoCCMINU;1718  case RISCV::MUL:   return RISCV::PseudoCCMUL;1719  case RISCV::LUI:   return RISCV::PseudoCCLUI;1720  case RISCV::QC_LI:   return RISCV::PseudoCCQC_LI;1721  case RISCV::QC_E_LI:   return RISCV::PseudoCCQC_E_LI;1722 1723  case RISCV::ADDI:  return RISCV::PseudoCCADDI;1724  case RISCV::SLLI:  return RISCV::PseudoCCSLLI;1725  case RISCV::SRLI:  return RISCV::PseudoCCSRLI;1726  case RISCV::SRAI:  return RISCV::PseudoCCSRAI;1727  case RISCV::ANDI:  return RISCV::PseudoCCANDI;1728  case RISCV::ORI:   return RISCV::PseudoCCORI;1729  case RISCV::XORI:  return RISCV::PseudoCCXORI;1730 1731  case RISCV::ADDW:  return RISCV::PseudoCCADDW;1732  case RISCV::SUBW:  return RISCV::PseudoCCSUBW;1733  case RISCV::SLLW:  return RISCV::PseudoCCSLLW;1734  case RISCV::SRLW:  return RISCV::PseudoCCSRLW;1735  case RISCV::SRAW:  return RISCV::PseudoCCSRAW;1736 1737  case RISCV::ADDIW: return RISCV::PseudoCCADDIW;1738  case RISCV::SLLIW: return RISCV::PseudoCCSLLIW;1739  case RISCV::SRLIW: return RISCV::PseudoCCSRLIW;1740  case RISCV::SRAIW: return RISCV::PseudoCCSRAIW;1741 1742  case RISCV::ANDN:  return RISCV::PseudoCCANDN;1743  case RISCV::ORN:   return RISCV::PseudoCCORN;1744  case RISCV::XNOR:  return RISCV::PseudoCCXNOR;1745 1746  case RISCV::NDS_BFOS:  return RISCV::PseudoCCNDS_BFOS;1747  case RISCV::NDS_BFOZ:  return RISCV::PseudoCCNDS_BFOZ;1748  }1749  // clang-format on1750 1751  return RISCV::INSTRUCTION_LIST_END;1752}1753 1754/// Identify instructions that can be folded into a CCMOV instruction, and1755/// return the defining instruction.1756static MachineInstr *canFoldAsPredicatedOp(Register Reg,1757                                           const MachineRegisterInfo &MRI,1758                                           const TargetInstrInfo *TII,1759                                           const RISCVSubtarget &STI) {1760  if (!Reg.isVirtual())1761    return nullptr;1762  if (!MRI.hasOneNonDBGUse(Reg))1763    return nullptr;1764  MachineInstr *MI = MRI.getVRegDef(Reg);1765  if (!MI)1766    return nullptr;1767 1768  if (!STI.hasShortForwardBranchIMinMax() &&1769      (MI->getOpcode() == RISCV::MAX || MI->getOpcode() == RISCV::MIN ||1770       MI->getOpcode() == RISCV::MINU || MI->getOpcode() == RISCV::MAXU))1771    return nullptr;1772 1773  if (!STI.hasShortForwardBranchIMul() && MI->getOpcode() == RISCV::MUL)1774    return nullptr;1775 1776  // Check if MI can be predicated and folded into the CCMOV.1777  if (getPredicatedOpcode(MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)1778    return nullptr;1779  // Don't predicate li idiom.1780  if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&1781      MI->getOperand(1).getReg() == RISCV::X0)1782    return nullptr;1783  // Check if MI has any other defs or physreg uses.1784  for (const MachineOperand &MO : llvm::drop_begin(MI->operands())) {1785    // Reject frame index operands, PEI can't handle the predicated pseudos.1786    if (MO.isFI() || MO.isCPI() || MO.isJTI())1787      return nullptr;1788    if (!MO.isReg())1789      continue;1790    // MI can't have any tied operands, that would conflict with predication.1791    if (MO.isTied())1792      return nullptr;1793    if (MO.isDef())1794      return nullptr;1795    // Allow constant physregs.1796    if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(MO.getReg()))1797      return nullptr;1798  }1799  bool DontMoveAcrossStores = true;1800  if (!MI->isSafeToMove(DontMoveAcrossStores))1801    return nullptr;1802  return MI;1803}1804 1805bool RISCVInstrInfo::analyzeSelect(const MachineInstr &MI,1806                                   SmallVectorImpl<MachineOperand> &Cond,1807                                   unsigned &TrueOp, unsigned &FalseOp,1808                                   bool &Optimizable) const {1809  assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&1810         "Unknown select instruction");1811  // CCMOV operands:1812  // 0: Def.1813  // 1: LHS of compare.1814  // 2: RHS of compare.1815  // 3: Condition code.1816  // 4: False use.1817  // 5: True use.1818  TrueOp = 5;1819  FalseOp = 4;1820  Cond.push_back(MI.getOperand(1));1821  Cond.push_back(MI.getOperand(2));1822  Cond.push_back(MI.getOperand(3));1823  // We can only fold when we support short forward branch opt.1824  Optimizable = STI.hasShortForwardBranchIALU();1825  return false;1826}1827 1828MachineInstr *1829RISCVInstrInfo::optimizeSelect(MachineInstr &MI,1830                               SmallPtrSetImpl<MachineInstr *> &SeenMIs,1831                               bool PreferFalse) const {1832  assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&1833         "Unknown select instruction");1834  if (!STI.hasShortForwardBranchIALU())1835    return nullptr;1836 1837  MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();1838  MachineInstr *DefMI =1839      canFoldAsPredicatedOp(MI.getOperand(5).getReg(), MRI, this, STI);1840  bool Invert = !DefMI;1841  if (!DefMI)1842    DefMI = canFoldAsPredicatedOp(MI.getOperand(4).getReg(), MRI, this, STI);1843  if (!DefMI)1844    return nullptr;1845 1846  // Find new register class to use.1847  MachineOperand FalseReg = MI.getOperand(Invert ? 5 : 4);1848  Register DestReg = MI.getOperand(0).getReg();1849  const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg());1850  if (!MRI.constrainRegClass(DestReg, PreviousClass))1851    return nullptr;1852 1853  unsigned PredOpc = getPredicatedOpcode(DefMI->getOpcode());1854  assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");1855 1856  // Create a new predicated version of DefMI.1857  MachineInstrBuilder NewMI =1858      BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(PredOpc), DestReg);1859 1860  // Copy the condition portion.1861  NewMI.add(MI.getOperand(1));1862  NewMI.add(MI.getOperand(2));1863 1864  // Add condition code, inverting if necessary.1865  auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());1866  if (Invert)1867    CC = RISCVCC::getInverseBranchCondition(CC);1868  NewMI.addImm(CC);1869 1870  // Copy the false register.1871  NewMI.add(FalseReg);1872 1873  // Copy all the DefMI operands.1874  const MCInstrDesc &DefDesc = DefMI->getDesc();1875  for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)1876    NewMI.add(DefMI->getOperand(i));1877 1878  // Update SeenMIs set: register newly created MI and erase removed DefMI.1879  SeenMIs.insert(NewMI);1880  SeenMIs.erase(DefMI);1881 1882  // If MI is inside a loop, and DefMI is outside the loop, then kill flags on1883  // DefMI would be invalid when transferred inside the loop.  Checking for a1884  // loop is expensive, but at least remove kill flags if they are in different1885  // BBs.1886  if (DefMI->getParent() != MI.getParent())1887    NewMI->clearKillInfo();1888 1889  // The caller will erase MI, but not DefMI.1890  DefMI->eraseFromParent();1891  return NewMI;1892}1893 1894unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {1895  if (MI.isMetaInstruction())1896    return 0;1897 1898  unsigned Opcode = MI.getOpcode();1899 1900  if (Opcode == TargetOpcode::INLINEASM ||1901      Opcode == TargetOpcode::INLINEASM_BR) {1902    const MachineFunction &MF = *MI.getParent()->getParent();1903    return getInlineAsmLength(MI.getOperand(0).getSymbolName(),1904                              *MF.getTarget().getMCAsmInfo());1905  }1906 1907  if (!MI.memoperands_empty()) {1908    MachineMemOperand *MMO = *(MI.memoperands_begin());1909    if (STI.hasStdExtZihintntl() && MMO->isNonTemporal()) {1910      if (STI.hasStdExtZca()) {1911        if (isCompressibleInst(MI, STI))1912          return 4; // c.ntl.all + c.load/c.store1913        return 6;   // c.ntl.all + load/store1914      }1915      return 8; // ntl.all + load/store1916    }1917  }1918 1919  if (Opcode == TargetOpcode::BUNDLE)1920    return getInstBundleLength(MI);1921 1922  if (MI.getParent() && MI.getParent()->getParent()) {1923    if (isCompressibleInst(MI, STI))1924      return 2;1925  }1926 1927  switch (Opcode) {1928  case RISCV::PseudoMV_FPR16INX:1929  case RISCV::PseudoMV_FPR32INX:1930    // MV is always compressible to either c.mv or c.li rd, 0.1931    return STI.hasStdExtZca() ? 2 : 4;1932  case TargetOpcode::STACKMAP:1933    // The upper bound for a stackmap intrinsic is the full length of its shadow1934    return StackMapOpers(&MI).getNumPatchBytes();1935  case TargetOpcode::PATCHPOINT:1936    // The size of the patchpoint intrinsic is the number of bytes requested1937    return PatchPointOpers(&MI).getNumPatchBytes();1938  case TargetOpcode::STATEPOINT: {1939    // The size of the statepoint intrinsic is the number of bytes requested1940    unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();1941    // No patch bytes means at most a PseudoCall is emitted1942    return std::max(NumBytes, 8U);1943  }1944  case TargetOpcode::PATCHABLE_FUNCTION_ENTER:1945  case TargetOpcode::PATCHABLE_FUNCTION_EXIT:1946  case TargetOpcode::PATCHABLE_TAIL_CALL: {1947    const MachineFunction &MF = *MI.getParent()->getParent();1948    const Function &F = MF.getFunction();1949    if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&1950        F.hasFnAttribute("patchable-function-entry")) {1951      unsigned Num;1952      if (F.getFnAttribute("patchable-function-entry")1953              .getValueAsString()1954              .getAsInteger(10, Num))1955        return get(Opcode).getSize();1956 1957      // Number of C.NOP or NOP1958      return (STI.hasStdExtZca() ? 2 : 4) * Num;1959    }1960    // XRay uses C.JAL + 21 or 33 C.NOP for each sled in RV32 and RV64,1961    // respectively.1962    return STI.is64Bit() ? 68 : 44;1963  }1964  default:1965    return get(Opcode).getSize();1966  }1967}1968 1969unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {1970  unsigned Size = 0;1971  MachineBasicBlock::const_instr_iterator I = MI.getIterator();1972  MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();1973  while (++I != E && I->isInsideBundle()) {1974    assert(!I->isBundle() && "No nested bundle!");1975    Size += getInstSizeInBytes(*I);1976  }1977  return Size;1978}1979 1980bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {1981  const unsigned Opcode = MI.getOpcode();1982  switch (Opcode) {1983  default:1984    break;1985  case RISCV::FSGNJ_D:1986  case RISCV::FSGNJ_S:1987  case RISCV::FSGNJ_H:1988  case RISCV::FSGNJ_D_INX:1989  case RISCV::FSGNJ_D_IN32X:1990  case RISCV::FSGNJ_S_INX:1991  case RISCV::FSGNJ_H_INX:1992    // The canonical floating-point move is fsgnj rd, rs, rs.1993    return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&1994           MI.getOperand(1).getReg() == MI.getOperand(2).getReg();1995  case RISCV::ADDI:1996  case RISCV::ORI:1997  case RISCV::XORI:1998    return (MI.getOperand(1).isReg() &&1999            MI.getOperand(1).getReg() == RISCV::X0) ||2000           (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0);2001  }2002  return MI.isAsCheapAsAMove();2003}2004 2005std::optional<DestSourcePair>2006RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {2007  if (MI.isMoveReg())2008    return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};2009  switch (MI.getOpcode()) {2010  default:2011    break;2012  case RISCV::ADD:2013  case RISCV::OR:2014  case RISCV::XOR:2015    if (MI.getOperand(1).isReg() && MI.getOperand(1).getReg() == RISCV::X0 &&2016        MI.getOperand(2).isReg())2017      return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};2018    if (MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0 &&2019        MI.getOperand(1).isReg())2020      return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};2021    break;2022  case RISCV::ADDI:2023    // Operand 1 can be a frameindex but callers expect registers2024    if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() &&2025        MI.getOperand(2).getImm() == 0)2026      return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};2027    break;2028  case RISCV::SUB:2029    if (MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0 &&2030        MI.getOperand(1).isReg())2031      return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};2032    break;2033  case RISCV::SH1ADD:2034  case RISCV::SH1ADD_UW:2035  case RISCV::SH2ADD:2036  case RISCV::SH2ADD_UW:2037  case RISCV::SH3ADD:2038  case RISCV::SH3ADD_UW:2039    if (MI.getOperand(1).isReg() && MI.getOperand(1).getReg() == RISCV::X0 &&2040        MI.getOperand(2).isReg())2041      return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};2042    break;2043  case RISCV::FSGNJ_D:2044  case RISCV::FSGNJ_S:2045  case RISCV::FSGNJ_H:2046  case RISCV::FSGNJ_D_INX:2047  case RISCV::FSGNJ_D_IN32X:2048  case RISCV::FSGNJ_S_INX:2049  case RISCV::FSGNJ_H_INX:2050    // The canonical floating-point move is fsgnj rd, rs, rs.2051    if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&2052        MI.getOperand(1).getReg() == MI.getOperand(2).getReg())2053      return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};2054    break;2055  }2056  return std::nullopt;2057}2058 2059MachineTraceStrategy RISCVInstrInfo::getMachineCombinerTraceStrategy() const {2060  if (ForceMachineCombinerStrategy.getNumOccurrences() == 0) {2061    // The option is unused. Choose Local strategy only for in-order cores. When2062    // scheduling model is unspecified, use MinInstrCount strategy as more2063    // generic one.2064    const auto &SchedModel = STI.getSchedModel();2065    return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())2066               ? MachineTraceStrategy::TS_MinInstrCount2067               : MachineTraceStrategy::TS_Local;2068  }2069  // The strategy was forced by the option.2070  return ForceMachineCombinerStrategy;2071}2072 2073void RISCVInstrInfo::finalizeInsInstrs(2074    MachineInstr &Root, unsigned &Pattern,2075    SmallVectorImpl<MachineInstr *> &InsInstrs) const {2076  int16_t FrmOpIdx =2077      RISCV::getNamedOperandIdx(Root.getOpcode(), RISCV::OpName::frm);2078  if (FrmOpIdx < 0) {2079    assert(all_of(InsInstrs,2080                  [](MachineInstr *MI) {2081                    return RISCV::getNamedOperandIdx(MI->getOpcode(),2082                                                     RISCV::OpName::frm) < 0;2083                  }) &&2084           "New instructions require FRM whereas the old one does not have it");2085    return;2086  }2087 2088  const MachineOperand &FRM = Root.getOperand(FrmOpIdx);2089  MachineFunction &MF = *Root.getMF();2090 2091  for (auto *NewMI : InsInstrs) {2092    // We'd already added the FRM operand.2093    if (static_cast<unsigned>(RISCV::getNamedOperandIdx(2094            NewMI->getOpcode(), RISCV::OpName::frm)) != NewMI->getNumOperands())2095      continue;2096    MachineInstrBuilder MIB(MF, NewMI);2097    MIB.add(FRM);2098    if (FRM.getImm() == RISCVFPRndMode::DYN)2099      MIB.addUse(RISCV::FRM, RegState::Implicit);2100  }2101}2102 2103static bool isFADD(unsigned Opc) {2104  switch (Opc) {2105  default:2106    return false;2107  case RISCV::FADD_H:2108  case RISCV::FADD_S:2109  case RISCV::FADD_D:2110    return true;2111  }2112}2113 2114static bool isFSUB(unsigned Opc) {2115  switch (Opc) {2116  default:2117    return false;2118  case RISCV::FSUB_H:2119  case RISCV::FSUB_S:2120  case RISCV::FSUB_D:2121    return true;2122  }2123}2124 2125static bool isFMUL(unsigned Opc) {2126  switch (Opc) {2127  default:2128    return false;2129  case RISCV::FMUL_H:2130  case RISCV::FMUL_S:2131  case RISCV::FMUL_D:2132    return true;2133  }2134}2135 2136bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,2137                                                       bool Invert) const {2138#define OPCODE_LMUL_CASE(OPC)                                                  \2139  case RISCV::OPC##_M1:                                                        \2140  case RISCV::OPC##_M2:                                                        \2141  case RISCV::OPC##_M4:                                                        \2142  case RISCV::OPC##_M8:                                                        \2143  case RISCV::OPC##_MF2:                                                       \2144  case RISCV::OPC##_MF4:                                                       \2145  case RISCV::OPC##_MF82146 2147#define OPCODE_LMUL_MASK_CASE(OPC)                                             \2148  case RISCV::OPC##_M1_MASK:                                                   \2149  case RISCV::OPC##_M2_MASK:                                                   \2150  case RISCV::OPC##_M4_MASK:                                                   \2151  case RISCV::OPC##_M8_MASK:                                                   \2152  case RISCV::OPC##_MF2_MASK:                                                  \2153  case RISCV::OPC##_MF4_MASK:                                                  \2154  case RISCV::OPC##_MF8_MASK2155 2156  unsigned Opcode = Inst.getOpcode();2157  if (Invert) {2158    if (auto InvOpcode = getInverseOpcode(Opcode))2159      Opcode = *InvOpcode;2160    else2161      return false;2162  }2163 2164  // clang-format off2165  switch (Opcode) {2166  default:2167    return false;2168  OPCODE_LMUL_CASE(PseudoVADD_VV):2169  OPCODE_LMUL_MASK_CASE(PseudoVADD_VV):2170  OPCODE_LMUL_CASE(PseudoVMUL_VV):2171  OPCODE_LMUL_MASK_CASE(PseudoVMUL_VV):2172    return true;2173  }2174  // clang-format on2175 2176#undef OPCODE_LMUL_MASK_CASE2177#undef OPCODE_LMUL_CASE2178}2179 2180bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,2181                                             const MachineInstr &Prev) const {2182  if (!areOpcodesEqualOrInverse(Root.getOpcode(), Prev.getOpcode()))2183    return false;2184 2185  assert(Root.getMF() == Prev.getMF());2186  const MachineRegisterInfo *MRI = &Root.getMF()->getRegInfo();2187  const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();2188 2189  // Make sure vtype operands are also the same.2190  const MCInstrDesc &Desc = get(Root.getOpcode());2191  const uint64_t TSFlags = Desc.TSFlags;2192 2193  auto checkImmOperand = [&](unsigned OpIdx) {2194    return Root.getOperand(OpIdx).getImm() == Prev.getOperand(OpIdx).getImm();2195  };2196 2197  auto checkRegOperand = [&](unsigned OpIdx) {2198    return Root.getOperand(OpIdx).getReg() == Prev.getOperand(OpIdx).getReg();2199  };2200 2201  // PassThru2202  // TODO: Potentially we can loosen the condition to consider Root to be2203  // associable with Prev if Root has NoReg as passthru. In which case we2204  // also need to loosen the condition on vector policies between these.2205  if (!checkRegOperand(1))2206    return false;2207 2208  // SEW2209  if (RISCVII::hasSEWOp(TSFlags) &&2210      !checkImmOperand(RISCVII::getSEWOpNum(Desc)))2211    return false;2212 2213  // Mask2214  if (RISCVII::usesMaskPolicy(TSFlags)) {2215    const MachineBasicBlock *MBB = Root.getParent();2216    const MachineBasicBlock::const_reverse_iterator It1(&Root);2217    const MachineBasicBlock::const_reverse_iterator It2(&Prev);2218    Register MI1VReg;2219 2220    bool SeenMI2 = false;2221    for (auto End = MBB->rend(), It = It1; It != End; ++It) {2222      if (It == It2) {2223        SeenMI2 = true;2224        if (!MI1VReg.isValid())2225          // There is no V0 def between Root and Prev; they're sharing the2226          // same V0.2227          break;2228      }2229 2230      if (It->modifiesRegister(RISCV::V0, TRI)) {2231        Register SrcReg = It->getOperand(1).getReg();2232        // If it's not VReg it'll be more difficult to track its defs, so2233        // bailing out here just to be safe.2234        if (!SrcReg.isVirtual())2235          return false;2236 2237        if (!MI1VReg.isValid()) {2238          // This is the V0 def for Root.2239          MI1VReg = SrcReg;2240          continue;2241        }2242 2243        // Some random mask updates.2244        if (!SeenMI2)2245          continue;2246 2247        // This is the V0 def for Prev; check if it's the same as that of2248        // Root.2249        if (MI1VReg != SrcReg)2250          return false;2251        else2252          break;2253      }2254    }2255 2256    // If we haven't encountered Prev, it's likely that this function was2257    // called in a wrong way (e.g. Root is before Prev).2258    assert(SeenMI2 && "Prev is expected to appear before Root");2259  }2260 2261  // Tail / Mask policies2262  if (RISCVII::hasVecPolicyOp(TSFlags) &&2263      !checkImmOperand(RISCVII::getVecPolicyOpNum(Desc)))2264    return false;2265 2266  // VL2267  if (RISCVII::hasVLOp(TSFlags)) {2268    unsigned OpIdx = RISCVII::getVLOpNum(Desc);2269    const MachineOperand &Op1 = Root.getOperand(OpIdx);2270    const MachineOperand &Op2 = Prev.getOperand(OpIdx);2271    if (Op1.getType() != Op2.getType())2272      return false;2273    switch (Op1.getType()) {2274    case MachineOperand::MO_Register:2275      if (Op1.getReg() != Op2.getReg())2276        return false;2277      break;2278    case MachineOperand::MO_Immediate:2279      if (Op1.getImm() != Op2.getImm())2280        return false;2281      break;2282    default:2283      llvm_unreachable("Unrecognized VL operand type");2284    }2285  }2286 2287  // Rounding modes2288  if (RISCVII::hasRoundModeOp(TSFlags) &&2289      !checkImmOperand(RISCVII::getVLOpNum(Desc) - 1))2290    return false;2291 2292  return true;2293}2294 2295// Most of our RVV pseudos have passthru operand, so the real operands2296// start from index = 2.2297bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,2298                                                  bool &Commuted) const {2299  const MachineBasicBlock *MBB = Inst.getParent();2300  const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();2301  assert(RISCVII::isFirstDefTiedToFirstUse(get(Inst.getOpcode())) &&2302         "Expect the present of passthrough operand.");2303  MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());2304  MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(3).getReg());2305 2306  // If only one operand has the same or inverse opcode and it's the second2307  // source operand, the operands must be commuted.2308  Commuted = !areRVVInstsReassociable(Inst, *MI1) &&2309             areRVVInstsReassociable(Inst, *MI2);2310  if (Commuted)2311    std::swap(MI1, MI2);2312 2313  return areRVVInstsReassociable(Inst, *MI1) &&2314         (isVectorAssociativeAndCommutative(*MI1) ||2315          isVectorAssociativeAndCommutative(*MI1, /* Invert */ true)) &&2316         hasReassociableOperands(*MI1, MBB) &&2317         MRI.hasOneNonDBGUse(MI1->getOperand(0).getReg());2318}2319 2320bool RISCVInstrInfo::hasReassociableOperands(2321    const MachineInstr &Inst, const MachineBasicBlock *MBB) const {2322  if (!isVectorAssociativeAndCommutative(Inst) &&2323      !isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))2324    return TargetInstrInfo::hasReassociableOperands(Inst, MBB);2325 2326  const MachineOperand &Op1 = Inst.getOperand(2);2327  const MachineOperand &Op2 = Inst.getOperand(3);2328  const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();2329 2330  // We need virtual register definitions for the operands that we will2331  // reassociate.2332  MachineInstr *MI1 = nullptr;2333  MachineInstr *MI2 = nullptr;2334  if (Op1.isReg() && Op1.getReg().isVirtual())2335    MI1 = MRI.getUniqueVRegDef(Op1.getReg());2336  if (Op2.isReg() && Op2.getReg().isVirtual())2337    MI2 = MRI.getUniqueVRegDef(Op2.getReg());2338 2339  // And at least one operand must be defined in MBB.2340  return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);2341}2342 2343void RISCVInstrInfo::getReassociateOperandIndices(2344    const MachineInstr &Root, unsigned Pattern,2345    std::array<unsigned, 5> &OperandIndices) const {2346  TargetInstrInfo::getReassociateOperandIndices(Root, Pattern, OperandIndices);2347  if (RISCV::getRVVMCOpcode(Root.getOpcode())) {2348    // Skip the passthrough operand, so increment all indices by one.2349    for (unsigned I = 0; I < 5; ++I)2350      ++OperandIndices[I];2351  }2352}2353 2354bool RISCVInstrInfo::hasReassociableSibling(const MachineInstr &Inst,2355                                            bool &Commuted) const {2356  if (isVectorAssociativeAndCommutative(Inst) ||2357      isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))2358    return hasReassociableVectorSibling(Inst, Commuted);2359 2360  if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))2361    return false;2362 2363  const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();2364  unsigned OperandIdx = Commuted ? 2 : 1;2365  const MachineInstr &Sibling =2366      *MRI.getVRegDef(Inst.getOperand(OperandIdx).getReg());2367 2368  int16_t InstFrmOpIdx =2369      RISCV::getNamedOperandIdx(Inst.getOpcode(), RISCV::OpName::frm);2370  int16_t SiblingFrmOpIdx =2371      RISCV::getNamedOperandIdx(Sibling.getOpcode(), RISCV::OpName::frm);2372 2373  return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||2374         RISCV::hasEqualFRM(Inst, Sibling);2375}2376 2377bool RISCVInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,2378                                                 bool Invert) const {2379  if (isVectorAssociativeAndCommutative(Inst, Invert))2380    return true;2381 2382  unsigned Opc = Inst.getOpcode();2383  if (Invert) {2384    auto InverseOpcode = getInverseOpcode(Opc);2385    if (!InverseOpcode)2386      return false;2387    Opc = *InverseOpcode;2388  }2389 2390  if (isFADD(Opc) || isFMUL(Opc))2391    return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&2392           Inst.getFlag(MachineInstr::MIFlag::FmNsz);2393 2394  switch (Opc) {2395  default:2396    return false;2397  case RISCV::ADD:2398  case RISCV::ADDW:2399  case RISCV::AND:2400  case RISCV::OR:2401  case RISCV::XOR:2402  // From RISC-V ISA spec, if both the high and low bits of the same product2403  // are required, then the recommended code sequence is:2404  //2405  // MULH[[S]U] rdh, rs1, rs22406  // MUL        rdl, rs1, rs22407  // (source register specifiers must be in same order and rdh cannot be the2408  //  same as rs1 or rs2)2409  //2410  // Microarchitectures can then fuse these into a single multiply operation2411  // instead of performing two separate multiplies.2412  // MachineCombiner may reassociate MUL operands and lose the fusion2413  // opportunity.2414  case RISCV::MUL:2415  case RISCV::MULW:2416  case RISCV::MIN:2417  case RISCV::MINU:2418  case RISCV::MAX:2419  case RISCV::MAXU:2420  case RISCV::FMIN_H:2421  case RISCV::FMIN_S:2422  case RISCV::FMIN_D:2423  case RISCV::FMAX_H:2424  case RISCV::FMAX_S:2425  case RISCV::FMAX_D:2426    return true;2427  }2428 2429  return false;2430}2431 2432std::optional<unsigned>2433RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {2434#define RVV_OPC_LMUL_CASE(OPC, INV)                                            \2435  case RISCV::OPC##_M1:                                                        \2436    return RISCV::INV##_M1;                                                    \2437  case RISCV::OPC##_M2:                                                        \2438    return RISCV::INV##_M2;                                                    \2439  case RISCV::OPC##_M4:                                                        \2440    return RISCV::INV##_M4;                                                    \2441  case RISCV::OPC##_M8:                                                        \2442    return RISCV::INV##_M8;                                                    \2443  case RISCV::OPC##_MF2:                                                       \2444    return RISCV::INV##_MF2;                                                   \2445  case RISCV::OPC##_MF4:                                                       \2446    return RISCV::INV##_MF4;                                                   \2447  case RISCV::OPC##_MF8:                                                       \2448    return RISCV::INV##_MF82449 2450#define RVV_OPC_LMUL_MASK_CASE(OPC, INV)                                       \2451  case RISCV::OPC##_M1_MASK:                                                   \2452    return RISCV::INV##_M1_MASK;                                               \2453  case RISCV::OPC##_M2_MASK:                                                   \2454    return RISCV::INV##_M2_MASK;                                               \2455  case RISCV::OPC##_M4_MASK:                                                   \2456    return RISCV::INV##_M4_MASK;                                               \2457  case RISCV::OPC##_M8_MASK:                                                   \2458    return RISCV::INV##_M8_MASK;                                               \2459  case RISCV::OPC##_MF2_MASK:                                                  \2460    return RISCV::INV##_MF2_MASK;                                              \2461  case RISCV::OPC##_MF4_MASK:                                                  \2462    return RISCV::INV##_MF4_MASK;                                              \2463  case RISCV::OPC##_MF8_MASK:                                                  \2464    return RISCV::INV##_MF8_MASK2465 2466  switch (Opcode) {2467  default:2468    return std::nullopt;2469  case RISCV::FADD_H:2470    return RISCV::FSUB_H;2471  case RISCV::FADD_S:2472    return RISCV::FSUB_S;2473  case RISCV::FADD_D:2474    return RISCV::FSUB_D;2475  case RISCV::FSUB_H:2476    return RISCV::FADD_H;2477  case RISCV::FSUB_S:2478    return RISCV::FADD_S;2479  case RISCV::FSUB_D:2480    return RISCV::FADD_D;2481  case RISCV::ADD:2482    return RISCV::SUB;2483  case RISCV::SUB:2484    return RISCV::ADD;2485  case RISCV::ADDW:2486    return RISCV::SUBW;2487  case RISCV::SUBW:2488    return RISCV::ADDW;2489    // clang-format off2490  RVV_OPC_LMUL_CASE(PseudoVADD_VV, PseudoVSUB_VV);2491  RVV_OPC_LMUL_MASK_CASE(PseudoVADD_VV, PseudoVSUB_VV);2492  RVV_OPC_LMUL_CASE(PseudoVSUB_VV, PseudoVADD_VV);2493  RVV_OPC_LMUL_MASK_CASE(PseudoVSUB_VV, PseudoVADD_VV);2494    // clang-format on2495  }2496 2497#undef RVV_OPC_LMUL_MASK_CASE2498#undef RVV_OPC_LMUL_CASE2499}2500 2501static bool canCombineFPFusedMultiply(const MachineInstr &Root,2502                                      const MachineOperand &MO,2503                                      bool DoRegPressureReduce) {2504  if (!MO.isReg() || !MO.getReg().isVirtual())2505    return false;2506  const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();2507  MachineInstr *MI = MRI.getVRegDef(MO.getReg());2508  if (!MI || !isFMUL(MI->getOpcode()))2509    return false;2510 2511  if (!Root.getFlag(MachineInstr::MIFlag::FmContract) ||2512      !MI->getFlag(MachineInstr::MIFlag::FmContract))2513    return false;2514 2515  // Try combining even if fmul has more than one use as it eliminates2516  // dependency between fadd(fsub) and fmul. However, it can extend liveranges2517  // for fmul operands, so reject the transformation in register pressure2518  // reduction mode.2519  if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))2520    return false;2521 2522  // Do not combine instructions from different basic blocks.2523  if (Root.getParent() != MI->getParent())2524    return false;2525  return RISCV::hasEqualFRM(Root, *MI);2526}2527 2528static bool getFPFusedMultiplyPatterns(MachineInstr &Root,2529                                       SmallVectorImpl<unsigned> &Patterns,2530                                       bool DoRegPressureReduce) {2531  unsigned Opc = Root.getOpcode();2532  bool IsFAdd = isFADD(Opc);2533  if (!IsFAdd && !isFSUB(Opc))2534    return false;2535  bool Added = false;2536  if (canCombineFPFusedMultiply(Root, Root.getOperand(1),2537                                DoRegPressureReduce)) {2538    Patterns.push_back(IsFAdd ? RISCVMachineCombinerPattern::FMADD_AX2539                              : RISCVMachineCombinerPattern::FMSUB);2540    Added = true;2541  }2542  if (canCombineFPFusedMultiply(Root, Root.getOperand(2),2543                                DoRegPressureReduce)) {2544    Patterns.push_back(IsFAdd ? RISCVMachineCombinerPattern::FMADD_XA2545                              : RISCVMachineCombinerPattern::FNMSUB);2546    Added = true;2547  }2548  return Added;2549}2550 2551static bool getFPPatterns(MachineInstr &Root,2552                          SmallVectorImpl<unsigned> &Patterns,2553                          bool DoRegPressureReduce) {2554  return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);2555}2556 2557/// Utility routine that checks if \param MO is defined by an2558/// \param CombineOpc instruction in the basic block \param MBB2559static const MachineInstr *canCombine(const MachineBasicBlock &MBB,2560                                      const MachineOperand &MO,2561                                      unsigned CombineOpc) {2562  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();2563  const MachineInstr *MI = nullptr;2564 2565  if (MO.isReg() && MO.getReg().isVirtual())2566    MI = MRI.getUniqueVRegDef(MO.getReg());2567  // And it needs to be in the trace (otherwise, it won't have a depth).2568  if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)2569    return nullptr;2570  // Must only used by the user we combine with.2571  if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))2572    return nullptr;2573 2574  return MI;2575}2576 2577/// Utility routine that checks if \param MO is defined by a SLLI in \param2578/// MBB that can be combined by splitting across 2 SHXADD instructions. The2579/// first SHXADD shift amount is given by \param OuterShiftAmt.2580static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB,2581                                      const MachineOperand &MO,2582                                      unsigned OuterShiftAmt) {2583  const MachineInstr *ShiftMI = canCombine(MBB, MO, RISCV::SLLI);2584  if (!ShiftMI)2585    return false;2586 2587  unsigned InnerShiftAmt = ShiftMI->getOperand(2).getImm();2588  if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)2589    return false;2590 2591  return true;2592}2593 2594// Returns the shift amount from a SHXADD instruction. Returns 0 if the2595// instruction is not a SHXADD.2596static unsigned getSHXADDShiftAmount(unsigned Opc) {2597  switch (Opc) {2598  default:2599    return 0;2600  case RISCV::SH1ADD:2601    return 1;2602  case RISCV::SH2ADD:2603    return 2;2604  case RISCV::SH3ADD:2605    return 3;2606  }2607}2608 2609// Returns the shift amount from a SHXADD.UW instruction. Returns 0 if the2610// instruction is not a SHXADD.UW.2611static unsigned getSHXADDUWShiftAmount(unsigned Opc) {2612  switch (Opc) {2613  default:2614    return 0;2615  case RISCV::SH1ADD_UW:2616    return 1;2617  case RISCV::SH2ADD_UW:2618    return 2;2619  case RISCV::SH3ADD_UW:2620    return 3;2621  }2622}2623 2624// Look for opportunities to combine (sh3add Z, (add X, (slli Y, 5))) into2625// (sh3add (sh2add Y, Z), X).2626static bool getSHXADDPatterns(const MachineInstr &Root,2627                              SmallVectorImpl<unsigned> &Patterns) {2628  unsigned ShiftAmt = getSHXADDShiftAmount(Root.getOpcode());2629  if (!ShiftAmt)2630    return false;2631 2632  const MachineBasicBlock &MBB = *Root.getParent();2633 2634  const MachineInstr *AddMI = canCombine(MBB, Root.getOperand(2), RISCV::ADD);2635  if (!AddMI)2636    return false;2637 2638  bool Found = false;2639  if (canCombineShiftIntoShXAdd(MBB, AddMI->getOperand(1), ShiftAmt)) {2640    Patterns.push_back(RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1);2641    Found = true;2642  }2643  if (canCombineShiftIntoShXAdd(MBB, AddMI->getOperand(2), ShiftAmt)) {2644    Patterns.push_back(RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2);2645    Found = true;2646  }2647 2648  return Found;2649}2650 2651CombinerObjective RISCVInstrInfo::getCombinerObjective(unsigned Pattern) const {2652  switch (Pattern) {2653  case RISCVMachineCombinerPattern::FMADD_AX:2654  case RISCVMachineCombinerPattern::FMADD_XA:2655  case RISCVMachineCombinerPattern::FMSUB:2656  case RISCVMachineCombinerPattern::FNMSUB:2657    return CombinerObjective::MustReduceDepth;2658  default:2659    return TargetInstrInfo::getCombinerObjective(Pattern);2660  }2661}2662 2663bool RISCVInstrInfo::getMachineCombinerPatterns(2664    MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,2665    bool DoRegPressureReduce) const {2666 2667  if (getFPPatterns(Root, Patterns, DoRegPressureReduce))2668    return true;2669 2670  if (getSHXADDPatterns(Root, Patterns))2671    return true;2672 2673  return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,2674                                                     DoRegPressureReduce);2675}2676 2677static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern) {2678  switch (RootOpc) {2679  default:2680    llvm_unreachable("Unexpected opcode");2681  case RISCV::FADD_H:2682    return RISCV::FMADD_H;2683  case RISCV::FADD_S:2684    return RISCV::FMADD_S;2685  case RISCV::FADD_D:2686    return RISCV::FMADD_D;2687  case RISCV::FSUB_H:2688    return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_H2689                                                         : RISCV::FNMSUB_H;2690  case RISCV::FSUB_S:2691    return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_S2692                                                         : RISCV::FNMSUB_S;2693  case RISCV::FSUB_D:2694    return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_D2695                                                         : RISCV::FNMSUB_D;2696  }2697}2698 2699static unsigned getAddendOperandIdx(unsigned Pattern) {2700  switch (Pattern) {2701  default:2702    llvm_unreachable("Unexpected pattern");2703  case RISCVMachineCombinerPattern::FMADD_AX:2704  case RISCVMachineCombinerPattern::FMSUB:2705    return 2;2706  case RISCVMachineCombinerPattern::FMADD_XA:2707  case RISCVMachineCombinerPattern::FNMSUB:2708    return 1;2709  }2710}2711 2712static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev,2713                                   unsigned Pattern,2714                                   SmallVectorImpl<MachineInstr *> &InsInstrs,2715                                   SmallVectorImpl<MachineInstr *> &DelInstrs) {2716  MachineFunction *MF = Root.getMF();2717  MachineRegisterInfo &MRI = MF->getRegInfo();2718  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();2719 2720  MachineOperand &Mul1 = Prev.getOperand(1);2721  MachineOperand &Mul2 = Prev.getOperand(2);2722  MachineOperand &Dst = Root.getOperand(0);2723  MachineOperand &Addend = Root.getOperand(getAddendOperandIdx(Pattern));2724 2725  Register DstReg = Dst.getReg();2726  unsigned FusedOpc = getFPFusedMultiplyOpcode(Root.getOpcode(), Pattern);2727  uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();2728  DebugLoc MergedLoc =2729      DILocation::getMergedLocation(Root.getDebugLoc(), Prev.getDebugLoc());2730 2731  bool Mul1IsKill = Mul1.isKill();2732  bool Mul2IsKill = Mul2.isKill();2733  bool AddendIsKill = Addend.isKill();2734 2735  // We need to clear kill flags since we may be extending the live range past2736  // a kill. If the mul had kill flags, we can preserve those since we know2737  // where the previous range stopped.2738  MRI.clearKillFlags(Mul1.getReg());2739  MRI.clearKillFlags(Mul2.getReg());2740 2741  MachineInstrBuilder MIB =2742      BuildMI(*MF, MergedLoc, TII->get(FusedOpc), DstReg)2743          .addReg(Mul1.getReg(), getKillRegState(Mul1IsKill))2744          .addReg(Mul2.getReg(), getKillRegState(Mul2IsKill))2745          .addReg(Addend.getReg(), getKillRegState(AddendIsKill))2746          .setMIFlags(IntersectedFlags);2747 2748  InsInstrs.push_back(MIB);2749  if (MRI.hasOneNonDBGUse(Prev.getOperand(0).getReg()))2750    DelInstrs.push_back(&Prev);2751  DelInstrs.push_back(&Root);2752}2753 2754// Combine patterns like (sh3add Z, (add X, (slli Y, 5))) to2755// (sh3add (sh2add Y, Z), X) if the shift amount can be split across two2756// shXadd instructions. The outer shXadd keeps its original opcode.2757static void2758genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,2759                  SmallVectorImpl<MachineInstr *> &InsInstrs,2760                  SmallVectorImpl<MachineInstr *> &DelInstrs,2761                  DenseMap<Register, unsigned> &InstrIdxForVirtReg) {2762  MachineFunction *MF = Root.getMF();2763  MachineRegisterInfo &MRI = MF->getRegInfo();2764  const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();2765 2766  unsigned OuterShiftAmt = getSHXADDShiftAmount(Root.getOpcode());2767  assert(OuterShiftAmt != 0 && "Unexpected opcode");2768 2769  MachineInstr *AddMI = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());2770  MachineInstr *ShiftMI =2771      MRI.getUniqueVRegDef(AddMI->getOperand(AddOpIdx).getReg());2772 2773  unsigned InnerShiftAmt = ShiftMI->getOperand(2).getImm();2774  assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");2775 2776  unsigned InnerOpc;2777  switch (InnerShiftAmt - OuterShiftAmt) {2778  default:2779    llvm_unreachable("Unexpected shift amount");2780  case 0:2781    InnerOpc = RISCV::ADD;2782    break;2783  case 1:2784    InnerOpc = RISCV::SH1ADD;2785    break;2786  case 2:2787    InnerOpc = RISCV::SH2ADD;2788    break;2789  case 3:2790    InnerOpc = RISCV::SH3ADD;2791    break;2792  }2793 2794  const MachineOperand &X = AddMI->getOperand(3 - AddOpIdx);2795  const MachineOperand &Y = ShiftMI->getOperand(1);2796  const MachineOperand &Z = Root.getOperand(1);2797 2798  Register NewVR = MRI.createVirtualRegister(&RISCV::GPRRegClass);2799 2800  auto MIB1 = BuildMI(*MF, MIMetadata(Root), TII->get(InnerOpc), NewVR)2801                  .addReg(Y.getReg(), getKillRegState(Y.isKill()))2802                  .addReg(Z.getReg(), getKillRegState(Z.isKill()));2803  auto MIB2 = BuildMI(*MF, MIMetadata(Root), TII->get(Root.getOpcode()),2804                      Root.getOperand(0).getReg())2805                  .addReg(NewVR, RegState::Kill)2806                  .addReg(X.getReg(), getKillRegState(X.isKill()));2807 2808  InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));2809  InsInstrs.push_back(MIB1);2810  InsInstrs.push_back(MIB2);2811  DelInstrs.push_back(ShiftMI);2812  DelInstrs.push_back(AddMI);2813  DelInstrs.push_back(&Root);2814}2815 2816void RISCVInstrInfo::genAlternativeCodeSequence(2817    MachineInstr &Root, unsigned Pattern,2818    SmallVectorImpl<MachineInstr *> &InsInstrs,2819    SmallVectorImpl<MachineInstr *> &DelInstrs,2820    DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {2821  MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();2822  switch (Pattern) {2823  default:2824    TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,2825                                                DelInstrs, InstrIdxForVirtReg);2826    return;2827  case RISCVMachineCombinerPattern::FMADD_AX:2828  case RISCVMachineCombinerPattern::FMSUB: {2829    MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(1).getReg());2830    combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);2831    return;2832  }2833  case RISCVMachineCombinerPattern::FMADD_XA:2834  case RISCVMachineCombinerPattern::FNMSUB: {2835    MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(2).getReg());2836    combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);2837    return;2838  }2839  case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1:2840    genShXAddAddShift(Root, 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);2841    return;2842  case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2:2843    genShXAddAddShift(Root, 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);2844    return;2845  }2846}2847 2848bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,2849                                       StringRef &ErrInfo) const {2850  MCInstrDesc const &Desc = MI.getDesc();2851 2852  for (const auto &[Index, Operand] : enumerate(Desc.operands())) {2853    unsigned OpType = Operand.OperandType;2854    if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&2855        OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) {2856      const MachineOperand &MO = MI.getOperand(Index);2857      if (MO.isReg()) {2858        ErrInfo = "Expected a non-register operand.";2859        return false;2860      }2861      if (MO.isImm()) {2862        int64_t Imm = MO.getImm();2863        bool Ok;2864        switch (OpType) {2865        default:2866          llvm_unreachable("Unexpected operand type");2867 2868          // clang-format off2869#define CASE_OPERAND_UIMM(NUM)                                                 \2870  case RISCVOp::OPERAND_UIMM##NUM:                                             \2871    Ok = isUInt<NUM>(Imm);                                                     \2872    break;2873#define CASE_OPERAND_SIMM(NUM)                                                 \2874  case RISCVOp::OPERAND_SIMM##NUM:                                             \2875    Ok = isInt<NUM>(Imm);                                                      \2876    break;2877        CASE_OPERAND_UIMM(1)2878        CASE_OPERAND_UIMM(2)2879        CASE_OPERAND_UIMM(3)2880        CASE_OPERAND_UIMM(4)2881        CASE_OPERAND_UIMM(5)2882        CASE_OPERAND_UIMM(6)2883        CASE_OPERAND_UIMM(7)2884        CASE_OPERAND_UIMM(8)2885        CASE_OPERAND_UIMM(9)2886        CASE_OPERAND_UIMM(10)2887        CASE_OPERAND_UIMM(12)2888        CASE_OPERAND_UIMM(16)2889        CASE_OPERAND_UIMM(20)2890        CASE_OPERAND_UIMM(32)2891        CASE_OPERAND_UIMM(48)2892        CASE_OPERAND_UIMM(64)2893          // clang-format on2894        case RISCVOp::OPERAND_UIMM2_LSB0:2895          Ok = isShiftedUInt<1, 1>(Imm);2896          break;2897        case RISCVOp::OPERAND_UIMM5_LSB0:2898          Ok = isShiftedUInt<4, 1>(Imm);2899          break;2900        case RISCVOp::OPERAND_UIMM5_NONZERO:2901          Ok = isUInt<5>(Imm) && (Imm != 0);2902          break;2903        case RISCVOp::OPERAND_UIMM5_GT3:2904          Ok = isUInt<5>(Imm) && (Imm > 3);2905          break;2906        case RISCVOp::OPERAND_UIMM5_PLUS1:2907          Ok = (isUInt<5>(Imm) && (Imm != 0)) || (Imm == 32);2908          break;2909        case RISCVOp::OPERAND_UIMM6_LSB0:2910          Ok = isShiftedUInt<5, 1>(Imm);2911          break;2912        case RISCVOp::OPERAND_UIMM7_LSB00:2913          Ok = isShiftedUInt<5, 2>(Imm);2914          break;2915        case RISCVOp::OPERAND_UIMM7_LSB000:2916          Ok = isShiftedUInt<4, 3>(Imm);2917          break;2918        case RISCVOp::OPERAND_UIMM8_LSB00:2919          Ok = isShiftedUInt<6, 2>(Imm);2920          break;2921        case RISCVOp::OPERAND_UIMM8_LSB000:2922          Ok = isShiftedUInt<5, 3>(Imm);2923          break;2924        case RISCVOp::OPERAND_UIMM8_GE32:2925          Ok = isUInt<8>(Imm) && Imm >= 32;2926          break;2927        case RISCVOp::OPERAND_UIMM9_LSB000:2928          Ok = isShiftedUInt<6, 3>(Imm);2929          break;2930        case RISCVOp::OPERAND_SIMM8_UNSIGNED:2931          Ok = isInt<8>(Imm);2932          break;2933        case RISCVOp::OPERAND_SIMM10_LSB0000_NONZERO:2934          Ok = isShiftedInt<6, 4>(Imm) && (Imm != 0);2935          break;2936        case RISCVOp::OPERAND_UIMM10_LSB00_NONZERO:2937          Ok = isShiftedUInt<8, 2>(Imm) && (Imm != 0);2938          break;2939        case RISCVOp::OPERAND_UIMM16_NONZERO:2940          Ok = isUInt<16>(Imm) && (Imm != 0);2941          break;2942        case RISCVOp::OPERAND_THREE:2943          Ok = Imm == 3;2944          break;2945        case RISCVOp::OPERAND_FOUR:2946          Ok = Imm == 4;2947          break;2948        case RISCVOp::OPERAND_IMM5_ZIBI:2949          Ok = (isUInt<5>(Imm) && Imm != 0) || Imm == -1;2950          break;2951          // clang-format off2952        CASE_OPERAND_SIMM(5)2953        CASE_OPERAND_SIMM(6)2954        CASE_OPERAND_SIMM(10)2955        CASE_OPERAND_SIMM(11)2956        CASE_OPERAND_SIMM(12)2957        CASE_OPERAND_SIMM(26)2958        // clang-format on2959        case RISCVOp::OPERAND_SIMM5_PLUS1:2960          Ok = (isInt<5>(Imm) && Imm != -16) || Imm == 16;2961          break;2962        case RISCVOp::OPERAND_SIMM5_NONZERO:2963          Ok = isInt<5>(Imm) && (Imm != 0);2964          break;2965        case RISCVOp::OPERAND_SIMM6_NONZERO:2966          Ok = Imm != 0 && isInt<6>(Imm);2967          break;2968        case RISCVOp::OPERAND_VTYPEI10:2969          Ok = isUInt<10>(Imm);2970          break;2971        case RISCVOp::OPERAND_VTYPEI11:2972          Ok = isUInt<11>(Imm);2973          break;2974        case RISCVOp::OPERAND_SIMM12_LSB00000:2975          Ok = isShiftedInt<7, 5>(Imm);2976          break;2977        case RISCVOp::OPERAND_SIMM16_NONZERO:2978          Ok = isInt<16>(Imm) && (Imm != 0);2979          break;2980        case RISCVOp::OPERAND_SIMM20_LI:2981          Ok = isInt<20>(Imm);2982          break;2983        case RISCVOp::OPERAND_BARE_SIMM32:2984          Ok = isInt<32>(Imm);2985          break;2986        case RISCVOp::OPERAND_UIMMLOG2XLEN:2987          Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);2988          break;2989        case RISCVOp::OPERAND_UIMMLOG2XLEN_NONZERO:2990          Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);2991          Ok = Ok && Imm != 0;2992          break;2993        case RISCVOp::OPERAND_CLUI_IMM:2994          Ok = (isUInt<5>(Imm) && Imm != 0) ||2995               (Imm >= 0xfffe0 && Imm <= 0xfffff);2996          break;2997        case RISCVOp::OPERAND_RVKRNUM:2998          Ok = Imm >= 0 && Imm <= 10;2999          break;3000        case RISCVOp::OPERAND_RVKRNUM_0_7:3001          Ok = Imm >= 0 && Imm <= 7;3002          break;3003        case RISCVOp::OPERAND_RVKRNUM_1_10:3004          Ok = Imm >= 1 && Imm <= 10;3005          break;3006        case RISCVOp::OPERAND_RVKRNUM_2_14:3007          Ok = Imm >= 2 && Imm <= 14;3008          break;3009        case RISCVOp::OPERAND_RLIST:3010          Ok = Imm >= RISCVZC::RA && Imm <= RISCVZC::RA_S0_S11;3011          break;3012        case RISCVOp::OPERAND_RLIST_S0:3013          Ok = Imm >= RISCVZC::RA_S0 && Imm <= RISCVZC::RA_S0_S11;3014          break;3015        case RISCVOp::OPERAND_STACKADJ:3016          Ok = Imm >= 0 && Imm <= 48 && Imm % 16 == 0;3017          break;3018        case RISCVOp::OPERAND_FRMARG:3019          Ok = RISCVFPRndMode::isValidRoundingMode(Imm);3020          break;3021        case RISCVOp::OPERAND_RTZARG:3022          Ok = Imm == RISCVFPRndMode::RTZ;3023          break;3024        case RISCVOp::OPERAND_COND_CODE:3025          Ok = Imm >= 0 && Imm < RISCVCC::COND_INVALID;3026          break;3027        case RISCVOp::OPERAND_VEC_POLICY:3028          Ok = (Imm &3029                (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) == Imm;3030          break;3031        case RISCVOp::OPERAND_SEW:3032          Ok = (isUInt<5>(Imm) && RISCVVType::isValidSEW(1 << Imm));3033          break;3034        case RISCVOp::OPERAND_SEW_MASK:3035          Ok = Imm == 0;3036          break;3037        case RISCVOp::OPERAND_VEC_RM:3038          assert(RISCVII::hasRoundModeOp(Desc.TSFlags));3039          if (RISCVII::usesVXRM(Desc.TSFlags))3040            Ok = isUInt<2>(Imm);3041          else3042            Ok = RISCVFPRndMode::isValidRoundingMode(Imm);3043          break;3044        case RISCVOp::OPERAND_XSFMM_VTYPE:3045          Ok = RISCVVType::isValidXSfmmVType(Imm);3046          break;3047        }3048        if (!Ok) {3049          ErrInfo = "Invalid immediate";3050          return false;3051        }3052      }3053    }3054  }3055 3056  const uint64_t TSFlags = Desc.TSFlags;3057  if (RISCVII::hasVLOp(TSFlags)) {3058    const MachineOperand &Op = MI.getOperand(RISCVII::getVLOpNum(Desc));3059    if (!Op.isImm() && !Op.isReg())  {3060      ErrInfo = "Invalid operand type for VL operand";3061      return false;3062    }3063    if (Op.isReg() && Op.getReg().isValid()) {3064      const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();3065      auto *RC = MRI.getRegClass(Op.getReg());3066      if (!RISCV::GPRRegClass.hasSubClassEq(RC)) {3067        ErrInfo = "Invalid register class for VL operand";3068        return false;3069      }3070    }3071    if (!RISCVII::hasSEWOp(TSFlags)) {3072      ErrInfo = "VL operand w/o SEW operand?";3073      return false;3074    }3075  }3076  if (RISCVII::hasSEWOp(TSFlags)) {3077    unsigned OpIdx = RISCVII::getSEWOpNum(Desc);3078    if (!MI.getOperand(OpIdx).isImm()) {3079      ErrInfo = "SEW value expected to be an immediate";3080      return false;3081    }3082    uint64_t Log2SEW = MI.getOperand(OpIdx).getImm();3083    if (Log2SEW > 31) {3084      ErrInfo = "Unexpected SEW value";3085      return false;3086    }3087    unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;3088    if (!RISCVVType::isValidSEW(SEW)) {3089      ErrInfo = "Unexpected SEW value";3090      return false;3091    }3092  }3093  if (RISCVII::hasVecPolicyOp(TSFlags)) {3094    unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);3095    if (!MI.getOperand(OpIdx).isImm()) {3096      ErrInfo = "Policy operand expected to be an immediate";3097      return false;3098    }3099    uint64_t Policy = MI.getOperand(OpIdx).getImm();3100    if (Policy > (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) {3101      ErrInfo = "Invalid Policy Value";3102      return false;3103    }3104    if (!RISCVII::hasVLOp(TSFlags)) {3105      ErrInfo = "policy operand w/o VL operand?";3106      return false;3107    }3108 3109    // VecPolicy operands can only exist on instructions with passthru/merge3110    // arguments. Note that not all arguments with passthru have vec policy3111    // operands- some instructions have implicit policies.3112    unsigned UseOpIdx;3113    if (!MI.isRegTiedToUseOperand(0, &UseOpIdx)) {3114      ErrInfo = "policy operand w/o tied operand?";3115      return false;3116    }3117  }3118 3119  if (int Idx = RISCVII::getFRMOpNum(Desc);3120      Idx >= 0 && MI.getOperand(Idx).getImm() == RISCVFPRndMode::DYN &&3121      !MI.readsRegister(RISCV::FRM, /*TRI=*/nullptr)) {3122    ErrInfo = "dynamic rounding mode should read FRM";3123    return false;3124  }3125 3126  return true;3127}3128 3129bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,3130                                         const MachineInstr &AddrI,3131                                         ExtAddrMode &AM) const {3132  switch (MemI.getOpcode()) {3133  default:3134    return false;3135  case RISCV::LB:3136  case RISCV::LBU:3137  case RISCV::LH:3138  case RISCV::LH_INX:3139  case RISCV::LHU:3140  case RISCV::LW:3141  case RISCV::LW_INX:3142  case RISCV::LWU:3143  case RISCV::LD:3144  case RISCV::LD_RV32:3145  case RISCV::FLH:3146  case RISCV::FLW:3147  case RISCV::FLD:3148  case RISCV::SB:3149  case RISCV::SH:3150  case RISCV::SH_INX:3151  case RISCV::SW:3152  case RISCV::SW_INX:3153  case RISCV::SD:3154  case RISCV::SD_RV32:3155  case RISCV::FSH:3156  case RISCV::FSW:3157  case RISCV::FSD:3158    break;3159  }3160 3161  if (MemI.getOperand(0).getReg() == Reg)3162    return false;3163 3164  if (AddrI.getOpcode() != RISCV::ADDI || !AddrI.getOperand(1).isReg() ||3165      !AddrI.getOperand(2).isImm())3166    return false;3167 3168  int64_t OldOffset = MemI.getOperand(2).getImm();3169  int64_t Disp = AddrI.getOperand(2).getImm();3170  int64_t NewOffset = OldOffset + Disp;3171  if (!STI.is64Bit())3172    NewOffset = SignExtend64<32>(NewOffset);3173 3174  if (!isInt<12>(NewOffset))3175    return false;3176 3177  AM.BaseReg = AddrI.getOperand(1).getReg();3178  AM.ScaledReg = 0;3179  AM.Scale = 0;3180  AM.Displacement = NewOffset;3181  AM.Form = ExtAddrMode::Formula::Basic;3182  return true;3183}3184 3185MachineInstr *RISCVInstrInfo::emitLdStWithAddr(MachineInstr &MemI,3186                                               const ExtAddrMode &AM) const {3187 3188  const DebugLoc &DL = MemI.getDebugLoc();3189  MachineBasicBlock &MBB = *MemI.getParent();3190 3191  assert(AM.ScaledReg == 0 && AM.Scale == 0 &&3192         "Addressing mode not supported for folding");3193 3194  return BuildMI(MBB, MemI, DL, get(MemI.getOpcode()))3195      .addReg(MemI.getOperand(0).getReg(),3196              MemI.mayLoad() ? RegState::Define : 0)3197      .addReg(AM.BaseReg)3198      .addImm(AM.Displacement)3199      .setMemRefs(MemI.memoperands())3200      .setMIFlags(MemI.getFlags());3201}3202 3203// TODO: At the moment, MIPS introduced paring of instructions operating with3204// word or double word. This should be extended with more instructions when more3205// vendors support load/store pairing.3206bool RISCVInstrInfo::isPairableLdStInstOpc(unsigned Opc) {3207  switch (Opc) {3208  default:3209    return false;3210  case RISCV::SW:3211  case RISCV::SD:3212  case RISCV::LD:3213  case RISCV::LW:3214    return true;3215  }3216}3217 3218bool RISCVInstrInfo::isLdStSafeToPair(const MachineInstr &LdSt,3219                                      const TargetRegisterInfo *TRI) {3220  // If this is a volatile load/store, don't mess with it.3221  if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)3222    return false;3223 3224  if (LdSt.getOperand(1).isFI())3225    return true;3226 3227  assert(LdSt.getOperand(1).isReg() && "Expected a reg operand.");3228  // Can't cluster if the instruction modifies the base register3229  // or it is update form. e.g. ld x5,8(x5)3230  if (LdSt.modifiesRegister(LdSt.getOperand(1).getReg(), TRI))3231    return false;3232 3233  if (!LdSt.getOperand(2).isImm())3234    return false;3235 3236  return true;3237}3238 3239bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(3240    const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,3241    int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,3242    const TargetRegisterInfo *TRI) const {3243  if (!LdSt.mayLoadOrStore())3244    return false;3245 3246  // Conservatively, only handle scalar loads/stores for now.3247  switch (LdSt.getOpcode()) {3248  case RISCV::LB:3249  case RISCV::LBU:3250  case RISCV::SB:3251  case RISCV::LH:3252  case RISCV::LH_INX:3253  case RISCV::LHU:3254  case RISCV::FLH:3255  case RISCV::SH:3256  case RISCV::SH_INX:3257  case RISCV::FSH:3258  case RISCV::LW:3259  case RISCV::LW_INX:3260  case RISCV::LWU:3261  case RISCV::FLW:3262  case RISCV::SW:3263  case RISCV::SW_INX:3264  case RISCV::FSW:3265  case RISCV::LD:3266  case RISCV::LD_RV32:3267  case RISCV::FLD:3268  case RISCV::SD:3269  case RISCV::SD_RV32:3270  case RISCV::FSD:3271    break;3272  default:3273    return false;3274  }3275  const MachineOperand *BaseOp;3276  OffsetIsScalable = false;3277  if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))3278    return false;3279  BaseOps.push_back(BaseOp);3280  return true;3281}3282 3283// TODO: This was copied from SIInstrInfo. Could it be lifted to a common3284// helper?3285static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,3286                                  ArrayRef<const MachineOperand *> BaseOps1,3287                                  const MachineInstr &MI2,3288                                  ArrayRef<const MachineOperand *> BaseOps2) {3289  // Only examine the first "base" operand of each instruction, on the3290  // assumption that it represents the real base address of the memory access.3291  // Other operands are typically offsets or indices from this base address.3292  if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))3293    return true;3294 3295  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())3296    return false;3297 3298  auto MO1 = *MI1.memoperands_begin();3299  auto MO2 = *MI2.memoperands_begin();3300  if (MO1->getAddrSpace() != MO2->getAddrSpace())3301    return false;3302 3303  auto Base1 = MO1->getValue();3304  auto Base2 = MO2->getValue();3305  if (!Base1 || !Base2)3306    return false;3307  Base1 = getUnderlyingObject(Base1);3308  Base2 = getUnderlyingObject(Base2);3309 3310  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))3311    return false;3312 3313  return Base1 == Base2;3314}3315 3316bool RISCVInstrInfo::shouldClusterMemOps(3317    ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1,3318    bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,3319    int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,3320    unsigned NumBytes) const {3321  // If the mem ops (to be clustered) do not have the same base ptr, then they3322  // should not be clustered3323  if (!BaseOps1.empty() && !BaseOps2.empty()) {3324    const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();3325    const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();3326    if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))3327      return false;3328  } else if (!BaseOps1.empty() || !BaseOps2.empty()) {3329    // If only one base op is empty, they do not have the same base ptr3330    return false;3331  }3332 3333  unsigned CacheLineSize =3334      BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();3335  // Assume a cache line size of 64 bytes if no size is set in RISCVSubtarget.3336  CacheLineSize = CacheLineSize ? CacheLineSize : 64;3337  // Cluster if the memory operations are on the same or a neighbouring cache3338  // line, but limit the maximum ClusterSize to avoid creating too much3339  // additional register pressure.3340  return ClusterSize <= 4 && std::abs(Offset1 - Offset2) < CacheLineSize;3341}3342 3343// Set BaseReg (the base register operand), Offset (the byte offset being3344// accessed) and the access Width of the passed instruction that reads/writes3345// memory. Returns false if the instruction does not read/write memory or the3346// BaseReg/Offset/Width can't be determined. Is not guaranteed to always3347// recognise base operands and offsets in all cases.3348// TODO: Add an IsScalable bool ref argument (like the equivalent AArch643349// function) and set it as appropriate.3350bool RISCVInstrInfo::getMemOperandWithOffsetWidth(3351    const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,3352    LocationSize &Width, const TargetRegisterInfo *TRI) const {3353  if (!LdSt.mayLoadOrStore())3354    return false;3355 3356  // Here we assume the standard RISC-V ISA, which uses a base+offset3357  // addressing mode. You'll need to relax these conditions to support custom3358  // load/store instructions.3359  if (LdSt.getNumExplicitOperands() != 3)3360    return false;3361  if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) ||3362      !LdSt.getOperand(2).isImm())3363    return false;3364 3365  if (!LdSt.hasOneMemOperand())3366    return false;3367 3368  Width = (*LdSt.memoperands_begin())->getSize();3369  BaseReg = &LdSt.getOperand(1);3370  Offset = LdSt.getOperand(2).getImm();3371  return true;3372}3373 3374bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(3375    const MachineInstr &MIa, const MachineInstr &MIb) const {3376  assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");3377  assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");3378 3379  if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||3380      MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())3381    return false;3382 3383  // Retrieve the base register, offset from the base register and width. Width3384  // is the size of memory that is being loaded/stored (e.g. 1, 2, 4).  If3385  // base registers are identical, and the offset of a lower memory access +3386  // the width doesn't overlap the offset of a higher memory access,3387  // then the memory accesses are different.3388  const TargetRegisterInfo *TRI = STI.getRegisterInfo();3389  const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;3390  int64_t OffsetA = 0, OffsetB = 0;3391  LocationSize WidthA = LocationSize::precise(0),3392               WidthB = LocationSize::precise(0);3393  if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&3394      getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {3395    if (BaseOpA->isIdenticalTo(*BaseOpB)) {3396      int LowOffset = std::min(OffsetA, OffsetB);3397      int HighOffset = std::max(OffsetA, OffsetB);3398      LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;3399      if (LowWidth.hasValue() &&3400          LowOffset + (int)LowWidth.getValue() <= HighOffset)3401        return true;3402    }3403  }3404  return false;3405}3406 3407std::pair<unsigned, unsigned>3408RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {3409  const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;3410  return std::make_pair(TF & Mask, TF & ~Mask);3411}3412 3413ArrayRef<std::pair<unsigned, const char *>>3414RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {3415  using namespace RISCVII;3416  static const std::pair<unsigned, const char *> TargetFlags[] = {3417      {MO_CALL, "riscv-call"},3418      {MO_LO, "riscv-lo"},3419      {MO_HI, "riscv-hi"},3420      {MO_PCREL_LO, "riscv-pcrel-lo"},3421      {MO_PCREL_HI, "riscv-pcrel-hi"},3422      {MO_GOT_HI, "riscv-got-hi"},3423      {MO_TPREL_LO, "riscv-tprel-lo"},3424      {MO_TPREL_HI, "riscv-tprel-hi"},3425      {MO_TPREL_ADD, "riscv-tprel-add"},3426      {MO_TLS_GOT_HI, "riscv-tls-got-hi"},3427      {MO_TLS_GD_HI, "riscv-tls-gd-hi"},3428      {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},3429      {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},3430      {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},3431      {MO_TLSDESC_CALL, "riscv-tlsdesc-call"}};3432  return ArrayRef(TargetFlags);3433}3434bool RISCVInstrInfo::isFunctionSafeToOutlineFrom(3435    MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {3436  const Function &F = MF.getFunction();3437 3438  // Can F be deduplicated by the linker? If it can, don't outline from it.3439  if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())3440    return false;3441 3442  // Don't outline from functions with section markings; the program could3443  // expect that all the code is in the named section.3444  if (F.hasSection())3445    return false;3446 3447  // It's safe to outline from MF.3448  return true;3449}3450 3451bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,3452                                            unsigned &Flags) const {3453  // More accurate safety checking is done in getOutliningCandidateInfo.3454  return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags);3455}3456 3457// Enum values indicating how an outlined call should be constructed.3458enum MachineOutlinerConstructionID {3459  MachineOutlinerTailCall,3460  MachineOutlinerDefault3461};3462 3463bool RISCVInstrInfo::shouldOutlineFromFunctionByDefault(3464    MachineFunction &MF) const {3465  return MF.getFunction().hasMinSize();3466}3467 3468static bool isCandidatePatchable(const MachineBasicBlock &MBB) {3469  const MachineFunction *MF = MBB.getParent();3470  const Function &F = MF->getFunction();3471  return F.getFnAttribute("fentry-call").getValueAsBool() ||3472         F.hasFnAttribute("patchable-function-entry");3473}3474 3475static bool isMIReadsReg(const MachineInstr &MI, const TargetRegisterInfo *TRI,3476                         MCRegister RegNo) {3477  return MI.readsRegister(RegNo, TRI) ||3478         MI.getDesc().hasImplicitUseOfPhysReg(RegNo);3479}3480 3481static bool isMIModifiesReg(const MachineInstr &MI,3482                            const TargetRegisterInfo *TRI, MCRegister RegNo) {3483  return MI.modifiesRegister(RegNo, TRI) ||3484         MI.getDesc().hasImplicitDefOfPhysReg(RegNo);3485}3486 3487static bool cannotInsertTailCall(const MachineBasicBlock &MBB) {3488  if (!MBB.back().isReturn())3489    return true;3490  if (isCandidatePatchable(MBB))3491    return true;3492 3493  // If the candidate reads the pre-set register3494  // that can be used for expanding PseudoTAIL instruction,3495  // then we cannot insert tail call.3496  const TargetSubtargetInfo &STI = MBB.getParent()->getSubtarget();3497  MCRegister TailExpandUseRegNo =3498      RISCVII::getTailExpandUseRegNo(STI.getFeatureBits());3499  for (const MachineInstr &MI : MBB) {3500    if (isMIReadsReg(MI, STI.getRegisterInfo(), TailExpandUseRegNo))3501      return true;3502    if (isMIModifiesReg(MI, STI.getRegisterInfo(), TailExpandUseRegNo))3503      break;3504  }3505  return false;3506}3507 3508static bool analyzeCandidate(outliner::Candidate &C) {3509  // If last instruction is return then we can rely on3510  // the verification already performed in the getOutliningTypeImpl.3511  if (C.back().isReturn()) {3512    assert(!cannotInsertTailCall(*C.getMBB()) &&3513           "The candidate who uses return instruction must be outlined "3514           "using tail call");3515    return false;3516  }3517 3518  // Filter out candidates where the X5 register (t0) can't be used to setup3519  // the function call.3520  const TargetRegisterInfo *TRI = C.getMF()->getSubtarget().getRegisterInfo();3521  if (llvm::any_of(C, [TRI](const MachineInstr &MI) {3522        return isMIModifiesReg(MI, TRI, RISCV::X5);3523      }))3524    return true;3525 3526  return !C.isAvailableAcrossAndOutOfSeq(RISCV::X5, *TRI);3527}3528 3529std::optional<std::unique_ptr<outliner::OutlinedFunction>>3530RISCVInstrInfo::getOutliningCandidateInfo(3531    const MachineModuleInfo &MMI,3532    std::vector<outliner::Candidate> &RepeatedSequenceLocs,3533    unsigned MinRepeats) const {3534 3535  // Analyze each candidate and erase the ones that are not viable.3536  llvm::erase_if(RepeatedSequenceLocs, analyzeCandidate);3537 3538  // If the sequence doesn't have enough candidates left, then we're done.3539  if (RepeatedSequenceLocs.size() < MinRepeats)3540    return std::nullopt;3541 3542  // Each RepeatedSequenceLoc is identical.3543  outliner::Candidate &Candidate = RepeatedSequenceLocs[0];3544  unsigned InstrSizeCExt =3545      Candidate.getMF()->getSubtarget<RISCVSubtarget>().hasStdExtZca() ? 2 : 4;3546  unsigned CallOverhead = 0, FrameOverhead = 0;3547 3548  // Count the number of CFI instructions in the candidate, if present.3549  unsigned CFICount = 0;3550  for (auto &I : Candidate) {3551    if (I.isCFIInstruction())3552      CFICount++;3553  }3554 3555  // Ensure CFI coverage matches: comparing the number of CFIs in the candidate3556  // with the total number of CFIs in the parent function for each candidate.3557  // Outlining only a subset of a function’s CFIs would split the unwind state3558  // across two code regions and lead to incorrect address offsets between the3559  // outlined body and the remaining code. To preserve correct unwind info, we3560  // only outline when all CFIs in the function can be outlined together.3561  for (outliner::Candidate &C : RepeatedSequenceLocs) {3562    std::vector<MCCFIInstruction> CFIInstructions =3563        C.getMF()->getFrameInstructions();3564 3565    if (CFICount > 0 && CFICount != CFIInstructions.size())3566      return std::nullopt;3567  }3568 3569  MachineOutlinerConstructionID MOCI = MachineOutlinerDefault;3570  if (Candidate.back().isReturn()) {3571    MOCI = MachineOutlinerTailCall;3572    // tail call = auipc + jalr in the worst case without linker relaxation.3573    // FIXME: This code suggests the JALR can be compressed - how?3574    CallOverhead = 4 + InstrSizeCExt;3575    // Using tail call we move ret instruction from caller to callee.3576    FrameOverhead = 0;3577  } else {3578    // call t0, function = 8 bytes.3579    CallOverhead = 8;3580    // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.3581    FrameOverhead = InstrSizeCExt;3582  }3583 3584  // If we have CFI instructions, we can only outline if the outlined section3585  // can be a tail call.3586  if (MOCI != MachineOutlinerTailCall && CFICount > 0)3587    return std::nullopt;3588 3589  for (auto &C : RepeatedSequenceLocs)3590    C.setCallInfo(MOCI, CallOverhead);3591 3592  unsigned SequenceSize = 0;3593  for (auto &MI : Candidate)3594    SequenceSize += getInstSizeInBytes(MI);3595 3596  return std::make_unique<outliner::OutlinedFunction>(3597      RepeatedSequenceLocs, SequenceSize, FrameOverhead, MOCI);3598}3599 3600outliner::InstrType3601RISCVInstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,3602                                     MachineBasicBlock::iterator &MBBI,3603                                     unsigned Flags) const {3604  MachineInstr &MI = *MBBI;3605  MachineBasicBlock *MBB = MI.getParent();3606  const TargetRegisterInfo *TRI =3607      MBB->getParent()->getSubtarget().getRegisterInfo();3608  const auto &F = MI.getMF()->getFunction();3609 3610  // We can only outline CFI instructions if we will tail call the outlined3611  // function, or fix up the CFI offsets. Currently, CFI instructions are3612  // outlined only if in a tail call.3613  if (MI.isCFIInstruction())3614    return outliner::InstrType::Legal;3615 3616  if (cannotInsertTailCall(*MBB) &&3617      (MI.isReturn() || isMIModifiesReg(MI, TRI, RISCV::X5)))3618    return outliner::InstrType::Illegal;3619 3620  // Make sure the operands don't reference something unsafe.3621  for (const auto &MO : MI.operands()) {3622 3623    // pcrel-hi and pcrel-lo can't put in separate sections, filter that out3624    // if any possible.3625    if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&3626        (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||3627         F.hasSection() || F.getSectionPrefix()))3628      return outliner::InstrType::Illegal;3629  }3630 3631  if (isLPAD(MI))3632    return outliner::InstrType::Illegal;3633 3634  return outliner::InstrType::Legal;3635}3636 3637void RISCVInstrInfo::buildOutlinedFrame(3638    MachineBasicBlock &MBB, MachineFunction &MF,3639    const outliner::OutlinedFunction &OF) const {3640 3641  if (OF.FrameConstructionID == MachineOutlinerTailCall)3642    return;3643 3644  MBB.addLiveIn(RISCV::X5);3645 3646  // Add in a return instruction to the end of the outlined frame.3647  MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR))3648      .addReg(RISCV::X0, RegState::Define)3649      .addReg(RISCV::X5)3650      .addImm(0));3651}3652 3653MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall(3654    Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,3655    MachineFunction &MF, outliner::Candidate &C) const {3656 3657  if (C.CallConstructionID == MachineOutlinerTailCall) {3658    It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(RISCV::PseudoTAIL))3659                            .addGlobalAddress(M.getNamedValue(MF.getName()),3660                                              /*Offset=*/0, RISCVII::MO_CALL));3661    return It;3662  }3663 3664  // Add in a call instruction to the outlined function at the given location.3665  It = MBB.insert(It,3666                  BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5)3667                      .addGlobalAddress(M.getNamedValue(MF.getName()), 0,3668                                        RISCVII::MO_CALL));3669  return It;3670}3671 3672std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,3673                                                         Register Reg) const {3674  // TODO: Handle cases where Reg is a super- or sub-register of the3675  // destination register.3676  const MachineOperand &Op0 = MI.getOperand(0);3677  if (!Op0.isReg() || Reg != Op0.getReg())3678    return std::nullopt;3679 3680  // Don't consider ADDIW as a candidate because the caller may not be aware3681  // of its sign extension behaviour.3682  if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(1).isReg() &&3683      MI.getOperand(2).isImm())3684    return RegImmPair{MI.getOperand(1).getReg(), MI.getOperand(2).getImm()};3685 3686  return std::nullopt;3687}3688 3689// MIR printer helper function to annotate Operands with a comment.3690std::string RISCVInstrInfo::createMIROperandComment(3691    const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,3692    const TargetRegisterInfo *TRI) const {3693  // Print a generic comment for this operand if there is one.3694  std::string GenericComment =3695      TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);3696  if (!GenericComment.empty())3697    return GenericComment;3698 3699  // If not, we must have an immediate operand.3700  if (!Op.isImm())3701    return std::string();3702 3703  const MCInstrDesc &Desc = MI.getDesc();3704  if (OpIdx >= Desc.getNumOperands())3705    return std::string();3706 3707  std::string Comment;3708  raw_string_ostream OS(Comment);3709 3710  const MCOperandInfo &OpInfo = Desc.operands()[OpIdx];3711 3712  // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW3713  // operand of vector codegen pseudos.3714  switch (OpInfo.OperandType) {3715  case RISCVOp::OPERAND_VTYPEI10:3716  case RISCVOp::OPERAND_VTYPEI11: {3717    unsigned Imm = Op.getImm();3718    RISCVVType::printVType(Imm, OS);3719    break;3720  }3721  case RISCVOp::OPERAND_XSFMM_VTYPE: {3722    unsigned Imm = Op.getImm();3723    RISCVVType::printXSfmmVType(Imm, OS);3724    break;3725  }3726  case RISCVOp::OPERAND_SEW:3727  case RISCVOp::OPERAND_SEW_MASK: {3728    unsigned Log2SEW = Op.getImm();3729    unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;3730    assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");3731    OS << "e" << SEW;3732    break;3733  }3734  case RISCVOp::OPERAND_VEC_POLICY:3735    unsigned Policy = Op.getImm();3736    assert(Policy <= (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC) &&3737           "Invalid Policy Value");3738    OS << (Policy & RISCVVType::TAIL_AGNOSTIC ? "ta" : "tu") << ", "3739       << (Policy & RISCVVType::MASK_AGNOSTIC ? "ma" : "mu");3740    break;3741  }3742 3743  return Comment;3744}3745 3746// clang-format off3747#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL)                                 \3748  RISCV::Pseudo##OP##_##LMUL3749 3750#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)                                   \3751  RISCV::Pseudo##OP##_##LMUL##_MASK3752 3753#define CASE_RVV_OPCODE_LMUL(OP, LMUL)                                        \3754  CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL):                                      \3755  case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)3756 3757#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP)                                      \3758  CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8):                                       \3759  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4):                                  \3760  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2):                                  \3761  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1):                                   \3762  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2):                                   \3763  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)3764 3765#define CASE_RVV_OPCODE_UNMASK(OP)                                            \3766  CASE_RVV_OPCODE_UNMASK_WIDEN(OP):                                           \3767  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)3768 3769#define CASE_RVV_OPCODE_MASK_WIDEN(OP)                                        \3770  CASE_RVV_OPCODE_MASK_LMUL(OP, MF8):                                         \3771  case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4):                                    \3772  case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2):                                    \3773  case CASE_RVV_OPCODE_MASK_LMUL(OP, M1):                                     \3774  case CASE_RVV_OPCODE_MASK_LMUL(OP, M2):                                     \3775  case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)3776 3777#define CASE_RVV_OPCODE_MASK(OP)                                              \3778  CASE_RVV_OPCODE_MASK_WIDEN(OP):                                             \3779  case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)3780 3781#define CASE_RVV_OPCODE_WIDEN(OP)                                             \3782  CASE_RVV_OPCODE_UNMASK_WIDEN(OP):                                           \3783  case CASE_RVV_OPCODE_MASK_WIDEN(OP)3784 3785#define CASE_RVV_OPCODE(OP)                                                   \3786  CASE_RVV_OPCODE_UNMASK(OP):                                                 \3787  case CASE_RVV_OPCODE_MASK(OP)3788// clang-format on3789 3790// clang-format off3791#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL)                                 \3792  RISCV::PseudoV##OP##_##TYPE##_##LMUL3793 3794#define CASE_VMA_OPCODE_LMULS(OP, TYPE)                                        \3795  CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8):                                       \3796  case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4):                                  \3797  case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2):                                  \3798  case CASE_VMA_OPCODE_COMMON(OP, TYPE, M1):                                   \3799  case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2):                                   \3800  case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4):                                   \3801  case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)3802 3803// VFMA instructions are SEW specific.3804#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW)                           \3805  RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW3806 3807#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)                               \3808  CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW):                                  \3809  case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW):                             \3810  case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW):                             \3811  case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)3812 3813#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)                              \3814  CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW):                                 \3815  case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)3816 3817#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW)                              \3818  CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW):                                 \3819  case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)3820 3821#define CASE_VFMA_OPCODE_VV(OP)                                                \3822  CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16):                                     \3823  case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VV, E16):                          \3824  case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32):                                \3825  case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)3826 3827#define CASE_VFMA_SPLATS(OP)                                                   \3828  CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16):                                 \3829  case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VFPR16, E16):                      \3830  case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32):                            \3831  case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)3832// clang-format on3833 3834bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,3835                                           unsigned &SrcOpIdx1,3836                                           unsigned &SrcOpIdx2) const {3837  const MCInstrDesc &Desc = MI.getDesc();3838  if (!Desc.isCommutable())3839    return false;3840 3841  switch (MI.getOpcode()) {3842  case RISCV::TH_MVEQZ:3843  case RISCV::TH_MVNEZ:3844    // We can't commute operands if operand 2 (i.e., rs1 in3845    // mveqz/mvnez rd,rs1,rs2) is the zero-register (as it is3846    // not valid as the in/out-operand 1).3847    if (MI.getOperand(2).getReg() == RISCV::X0)3848      return false;3849    // Operands 1 and 2 are commutable, if we switch the opcode.3850    return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2);3851  case RISCV::QC_SELECTIEQ:3852  case RISCV::QC_SELECTINE:3853  case RISCV::QC_SELECTIIEQ:3854  case RISCV::QC_SELECTIINE:3855    return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2);3856  case RISCV::QC_MVEQ:3857  case RISCV::QC_MVNE:3858  case RISCV::QC_MVLT:3859  case RISCV::QC_MVGE:3860  case RISCV::QC_MVLTU:3861  case RISCV::QC_MVGEU:3862  case RISCV::QC_MVEQI:3863  case RISCV::QC_MVNEI:3864  case RISCV::QC_MVLTI:3865  case RISCV::QC_MVGEI:3866  case RISCV::QC_MVLTUI:3867  case RISCV::QC_MVGEUI:3868    return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 4);3869  case RISCV::TH_MULA:3870  case RISCV::TH_MULAW:3871  case RISCV::TH_MULAH:3872  case RISCV::TH_MULS:3873  case RISCV::TH_MULSW:3874  case RISCV::TH_MULSH:3875    // Operands 2 and 3 are commutable.3876    return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);3877  case RISCV::PseudoCCMOVGPRNoX0:3878  case RISCV::PseudoCCMOVGPR:3879    // Operands 4 and 5 are commutable.3880    return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 4, 5);3881  case CASE_RVV_OPCODE(VADD_VV):3882  case CASE_RVV_OPCODE(VAND_VV):3883  case CASE_RVV_OPCODE(VOR_VV):3884  case CASE_RVV_OPCODE(VXOR_VV):3885  case CASE_RVV_OPCODE_MASK(VMSEQ_VV):3886  case CASE_RVV_OPCODE_MASK(VMSNE_VV):3887  case CASE_RVV_OPCODE(VMIN_VV):3888  case CASE_RVV_OPCODE(VMINU_VV):3889  case CASE_RVV_OPCODE(VMAX_VV):3890  case CASE_RVV_OPCODE(VMAXU_VV):3891  case CASE_RVV_OPCODE(VMUL_VV):3892  case CASE_RVV_OPCODE(VMULH_VV):3893  case CASE_RVV_OPCODE(VMULHU_VV):3894  case CASE_RVV_OPCODE_WIDEN(VWADD_VV):3895  case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):3896  case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):3897  case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):3898  case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):3899  case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):3900  case CASE_RVV_OPCODE_UNMASK(VADC_VVM):3901  case CASE_RVV_OPCODE(VSADD_VV):3902  case CASE_RVV_OPCODE(VSADDU_VV):3903  case CASE_RVV_OPCODE(VAADD_VV):3904  case CASE_RVV_OPCODE(VAADDU_VV):3905  case CASE_RVV_OPCODE(VSMUL_VV):3906    // Operands 2 and 3 are commutable.3907    return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);3908  case CASE_VFMA_SPLATS(FMADD):3909  case CASE_VFMA_SPLATS(FMSUB):3910  case CASE_VFMA_SPLATS(FMACC):3911  case CASE_VFMA_SPLATS(FMSAC):3912  case CASE_VFMA_SPLATS(FNMADD):3913  case CASE_VFMA_SPLATS(FNMSUB):3914  case CASE_VFMA_SPLATS(FNMACC):3915  case CASE_VFMA_SPLATS(FNMSAC):3916  case CASE_VFMA_OPCODE_VV(FMACC):3917  case CASE_VFMA_OPCODE_VV(FMSAC):3918  case CASE_VFMA_OPCODE_VV(FNMACC):3919  case CASE_VFMA_OPCODE_VV(FNMSAC):3920  case CASE_VMA_OPCODE_LMULS(MADD, VX):3921  case CASE_VMA_OPCODE_LMULS(NMSUB, VX):3922  case CASE_VMA_OPCODE_LMULS(MACC, VX):3923  case CASE_VMA_OPCODE_LMULS(NMSAC, VX):3924  case CASE_VMA_OPCODE_LMULS(MACC, VV):3925  case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {3926    // If the tail policy is undisturbed we can't commute.3927    assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));3928    if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &3929         1) == 0)3930      return false;3931 3932    // For these instructions we can only swap operand 1 and operand 3 by3933    // changing the opcode.3934    unsigned CommutableOpIdx1 = 1;3935    unsigned CommutableOpIdx2 = 3;3936    if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,3937                              CommutableOpIdx2))3938      return false;3939    return true;3940  }3941  case CASE_VFMA_OPCODE_VV(FMADD):3942  case CASE_VFMA_OPCODE_VV(FMSUB):3943  case CASE_VFMA_OPCODE_VV(FNMADD):3944  case CASE_VFMA_OPCODE_VV(FNMSUB):3945  case CASE_VMA_OPCODE_LMULS(MADD, VV):3946  case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {3947    // If the tail policy is undisturbed we can't commute.3948    assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));3949    if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &3950         1) == 0)3951      return false;3952 3953    // For these instructions we have more freedom. We can commute with the3954    // other multiplicand or with the addend/subtrahend/minuend.3955 3956    // Any fixed operand must be from source 1, 2 or 3.3957    if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)3958      return false;3959    if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)3960      return false;3961 3962    // It both ops are fixed one must be the tied source.3963    if (SrcOpIdx1 != CommuteAnyOperandIndex &&3964        SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)3965      return false;3966 3967    // Look for two different register operands assumed to be commutable3968    // regardless of the FMA opcode. The FMA opcode is adjusted later if3969    // needed.3970    if (SrcOpIdx1 == CommuteAnyOperandIndex ||3971        SrcOpIdx2 == CommuteAnyOperandIndex) {3972      // At least one of operands to be commuted is not specified and3973      // this method is free to choose appropriate commutable operands.3974      unsigned CommutableOpIdx1 = SrcOpIdx1;3975      if (SrcOpIdx1 == SrcOpIdx2) {3976        // Both of operands are not fixed. Set one of commutable3977        // operands to the tied source.3978        CommutableOpIdx1 = 1;3979      } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {3980        // Only one of the operands is not fixed.3981        CommutableOpIdx1 = SrcOpIdx2;3982      }3983 3984      // CommutableOpIdx1 is well defined now. Let's choose another commutable3985      // operand and assign its index to CommutableOpIdx2.3986      unsigned CommutableOpIdx2;3987      if (CommutableOpIdx1 != 1) {3988        // If we haven't already used the tied source, we must use it now.3989        CommutableOpIdx2 = 1;3990      } else {3991        Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg();3992 3993        // The commuted operands should have different registers.3994        // Otherwise, the commute transformation does not change anything and3995        // is useless. We use this as a hint to make our decision.3996        if (Op1Reg != MI.getOperand(2).getReg())3997          CommutableOpIdx2 = 2;3998        else3999          CommutableOpIdx2 = 3;4000      }4001 4002      // Assign the found pair of commutable indices to SrcOpIdx1 and4003      // SrcOpIdx2 to return those values.4004      if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,4005                                CommutableOpIdx2))4006        return false;4007    }4008 4009    return true;4010  }4011  }4012 4013  return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);4014}4015 4016// clang-format off4017#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL)                \4018  case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL:                                \4019    Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL;                             \4020    break;4021 4022#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)                       \4023  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8)                       \4024  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4)                       \4025  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2)                       \4026  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1)                        \4027  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2)                        \4028  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4)                        \4029  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)4030 4031// VFMA depends on SEW.4032#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW)          \4033  case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW:                        \4034    Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW;                     \4035    break;4036 4037#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)              \4038  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW)                  \4039  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW)                  \4040  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW)                  \4041  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)4042 4043#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)             \4044  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW)                 \4045  CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)4046 4047#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW)             \4048  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW)                 \4049  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)4050 4051#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)                               \4052  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16)                     \4053  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VV, E16)         \4054  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32)                     \4055  CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)4056 4057#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)                           \4058  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16)                 \4059  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VFPR16, E16)     \4060  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32)                 \4061  CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)4062// clang-format on4063 4064MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,4065                                                     bool NewMI,4066                                                     unsigned OpIdx1,4067                                                     unsigned OpIdx2) const {4068  auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {4069    if (NewMI)4070      return *MI.getParent()->getParent()->CloneMachineInstr(&MI);4071    return MI;4072  };4073 4074  switch (MI.getOpcode()) {4075  case RISCV::TH_MVEQZ:4076  case RISCV::TH_MVNEZ: {4077    auto &WorkingMI = cloneIfNew(MI);4078    WorkingMI.setDesc(get(MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ4079                                                            : RISCV::TH_MVEQZ));4080    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, false, OpIdx1,4081                                                   OpIdx2);4082  }4083  case RISCV::QC_SELECTIEQ:4084  case RISCV::QC_SELECTINE:4085  case RISCV::QC_SELECTIIEQ:4086  case RISCV::QC_SELECTIINE:4087    return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);4088  case RISCV::QC_MVEQ:4089  case RISCV::QC_MVNE:4090  case RISCV::QC_MVLT:4091  case RISCV::QC_MVGE:4092  case RISCV::QC_MVLTU:4093  case RISCV::QC_MVGEU:4094  case RISCV::QC_MVEQI:4095  case RISCV::QC_MVNEI:4096  case RISCV::QC_MVLTI:4097  case RISCV::QC_MVGEI:4098  case RISCV::QC_MVLTUI:4099  case RISCV::QC_MVGEUI: {4100    auto &WorkingMI = cloneIfNew(MI);4101    WorkingMI.setDesc(get(getInverseXqcicmOpcode(MI.getOpcode())));4102    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, false, OpIdx1,4103                                                   OpIdx2);4104  }4105  case RISCV::PseudoCCMOVGPRNoX0:4106  case RISCV::PseudoCCMOVGPR: {4107    // CCMOV can be commuted by inverting the condition.4108    auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());4109    CC = RISCVCC::getInverseBranchCondition(CC);4110    auto &WorkingMI = cloneIfNew(MI);4111    WorkingMI.getOperand(3).setImm(CC);4112    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI*/ false,4113                                                   OpIdx1, OpIdx2);4114  }4115  case CASE_VFMA_SPLATS(FMACC):4116  case CASE_VFMA_SPLATS(FMADD):4117  case CASE_VFMA_SPLATS(FMSAC):4118  case CASE_VFMA_SPLATS(FMSUB):4119  case CASE_VFMA_SPLATS(FNMACC):4120  case CASE_VFMA_SPLATS(FNMADD):4121  case CASE_VFMA_SPLATS(FNMSAC):4122  case CASE_VFMA_SPLATS(FNMSUB):4123  case CASE_VFMA_OPCODE_VV(FMACC):4124  case CASE_VFMA_OPCODE_VV(FMSAC):4125  case CASE_VFMA_OPCODE_VV(FNMACC):4126  case CASE_VFMA_OPCODE_VV(FNMSAC):4127  case CASE_VMA_OPCODE_LMULS(MADD, VX):4128  case CASE_VMA_OPCODE_LMULS(NMSUB, VX):4129  case CASE_VMA_OPCODE_LMULS(MACC, VX):4130  case CASE_VMA_OPCODE_LMULS(NMSAC, VX):4131  case CASE_VMA_OPCODE_LMULS(MACC, VV):4132  case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {4133    // It only make sense to toggle these between clobbering the4134    // addend/subtrahend/minuend one of the multiplicands.4135    assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");4136    assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");4137    unsigned Opc;4138    switch (MI.getOpcode()) {4139      default:4140        llvm_unreachable("Unexpected opcode");4141      CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)4142      CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)4143      CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB)4144      CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC)4145      CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD)4146      CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)4147      CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB)4148      CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC)4149      CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)4150      CASE_VFMA_CHANGE_OPCODE_VV(FMSAC, FMSUB)4151      CASE_VFMA_CHANGE_OPCODE_VV(FNMACC, FNMADD)4152      CASE_VFMA_CHANGE_OPCODE_VV(FNMSAC, FNMSUB)4153      CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)4154      CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)4155      CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)4156      CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)4157      CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)4158      CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)4159    }4160 4161    auto &WorkingMI = cloneIfNew(MI);4162    WorkingMI.setDesc(get(Opc));4163    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,4164                                                   OpIdx1, OpIdx2);4165  }4166  case CASE_VFMA_OPCODE_VV(FMADD):4167  case CASE_VFMA_OPCODE_VV(FMSUB):4168  case CASE_VFMA_OPCODE_VV(FNMADD):4169  case CASE_VFMA_OPCODE_VV(FNMSUB):4170  case CASE_VMA_OPCODE_LMULS(MADD, VV):4171  case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {4172    assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");4173    // If one of the operands, is the addend we need to change opcode.4174    // Otherwise we're just swapping 2 of the multiplicands.4175    if (OpIdx1 == 3 || OpIdx2 == 3) {4176      unsigned Opc;4177      switch (MI.getOpcode()) {4178        default:4179          llvm_unreachable("Unexpected opcode");4180        CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)4181        CASE_VFMA_CHANGE_OPCODE_VV(FMSUB, FMSAC)4182        CASE_VFMA_CHANGE_OPCODE_VV(FNMADD, FNMACC)4183        CASE_VFMA_CHANGE_OPCODE_VV(FNMSUB, FNMSAC)4184        CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)4185        CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)4186      }4187 4188      auto &WorkingMI = cloneIfNew(MI);4189      WorkingMI.setDesc(get(Opc));4190      return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,4191                                                     OpIdx1, OpIdx2);4192    }4193    // Let the default code handle it.4194    break;4195  }4196  }4197 4198  return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);4199}4200 4201#undef CASE_VMA_CHANGE_OPCODE_COMMON4202#undef CASE_VMA_CHANGE_OPCODE_LMULS4203#undef CASE_VFMA_CHANGE_OPCODE_COMMON4204#undef CASE_VFMA_CHANGE_OPCODE_LMULS_M14205#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF24206#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF44207#undef CASE_VFMA_CHANGE_OPCODE_VV4208#undef CASE_VFMA_CHANGE_OPCODE_SPLATS4209 4210#undef CASE_RVV_OPCODE_UNMASK_LMUL4211#undef CASE_RVV_OPCODE_MASK_LMUL4212#undef CASE_RVV_OPCODE_LMUL4213#undef CASE_RVV_OPCODE_UNMASK_WIDEN4214#undef CASE_RVV_OPCODE_UNMASK4215#undef CASE_RVV_OPCODE_MASK_WIDEN4216#undef CASE_RVV_OPCODE_MASK4217#undef CASE_RVV_OPCODE_WIDEN4218#undef CASE_RVV_OPCODE4219 4220#undef CASE_VMA_OPCODE_COMMON4221#undef CASE_VMA_OPCODE_LMULS4222#undef CASE_VFMA_OPCODE_COMMON4223#undef CASE_VFMA_OPCODE_LMULS_M14224#undef CASE_VFMA_OPCODE_LMULS_MF24225#undef CASE_VFMA_OPCODE_LMULS_MF44226#undef CASE_VFMA_OPCODE_VV4227#undef CASE_VFMA_SPLATS4228 4229bool RISCVInstrInfo::simplifyInstruction(MachineInstr &MI) const {4230  switch (MI.getOpcode()) {4231  default:4232    break;4233  case RISCV::ADD:4234  case RISCV::OR:4235  case RISCV::XOR:4236    // Normalize (so we hit the next if clause).4237    // add/[x]or rd, zero, rs => add/[x]or rd, rs, zero4238    if (MI.getOperand(1).getReg() == RISCV::X0)4239      commuteInstruction(MI);4240    // add/[x]or rd, rs, zero => addi rd, rs, 04241    if (MI.getOperand(2).getReg() == RISCV::X0) {4242      MI.getOperand(2).ChangeToImmediate(0);4243      MI.setDesc(get(RISCV::ADDI));4244      return true;4245    }4246    // xor rd, rs, rs => addi rd, zero, 04247    if (MI.getOpcode() == RISCV::XOR &&4248        MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {4249      MI.getOperand(1).setReg(RISCV::X0);4250      MI.getOperand(2).ChangeToImmediate(0);4251      MI.setDesc(get(RISCV::ADDI));4252      return true;4253    }4254    break;4255  case RISCV::ORI:4256  case RISCV::XORI:4257    // [x]ori rd, zero, N => addi rd, zero, N4258    if (MI.getOperand(1).getReg() == RISCV::X0) {4259      MI.setDesc(get(RISCV::ADDI));4260      return true;4261    }4262    break;4263  case RISCV::SUB:4264    // sub rd, rs, zero => addi rd, rs, 04265    if (MI.getOperand(2).getReg() == RISCV::X0) {4266      MI.getOperand(2).ChangeToImmediate(0);4267      MI.setDesc(get(RISCV::ADDI));4268      return true;4269    }4270    break;4271  case RISCV::SUBW:4272    // subw rd, rs, zero => addiw rd, rs, 04273    if (MI.getOperand(2).getReg() == RISCV::X0) {4274      MI.getOperand(2).ChangeToImmediate(0);4275      MI.setDesc(get(RISCV::ADDIW));4276      return true;4277    }4278    break;4279  case RISCV::ADDW:4280    // Normalize (so we hit the next if clause).4281    // addw rd, zero, rs => addw rd, rs, zero4282    if (MI.getOperand(1).getReg() == RISCV::X0)4283      commuteInstruction(MI);4284    // addw rd, rs, zero => addiw rd, rs, 04285    if (MI.getOperand(2).getReg() == RISCV::X0) {4286      MI.getOperand(2).ChangeToImmediate(0);4287      MI.setDesc(get(RISCV::ADDIW));4288      return true;4289    }4290    break;4291  case RISCV::SH1ADD:4292  case RISCV::SH1ADD_UW:4293  case RISCV::SH2ADD:4294  case RISCV::SH2ADD_UW:4295  case RISCV::SH3ADD:4296  case RISCV::SH3ADD_UW:4297    // shNadd[.uw] rd, zero, rs => addi rd, rs, 04298    if (MI.getOperand(1).getReg() == RISCV::X0) {4299      MI.removeOperand(1);4300      MI.addOperand(MachineOperand::CreateImm(0));4301      MI.setDesc(get(RISCV::ADDI));4302      return true;4303    }4304    // shNadd[.uw] rd, rs, zero => slli[.uw] rd, rs, N4305    if (MI.getOperand(2).getReg() == RISCV::X0) {4306      MI.removeOperand(2);4307      unsigned Opc = MI.getOpcode();4308      if (Opc == RISCV::SH1ADD_UW || Opc == RISCV::SH2ADD_UW ||4309          Opc == RISCV::SH3ADD_UW) {4310        MI.addOperand(MachineOperand::CreateImm(getSHXADDUWShiftAmount(Opc)));4311        MI.setDesc(get(RISCV::SLLI_UW));4312        return true;4313      }4314      MI.addOperand(MachineOperand::CreateImm(getSHXADDShiftAmount(Opc)));4315      MI.setDesc(get(RISCV::SLLI));4316      return true;4317    }4318    break;4319  case RISCV::AND:4320  case RISCV::MUL:4321  case RISCV::MULH:4322  case RISCV::MULHSU:4323  case RISCV::MULHU:4324  case RISCV::MULW:4325    // and rd, zero, rs => addi rd, zero, 04326    // mul* rd, zero, rs => addi rd, zero, 04327    // and rd, rs, zero => addi rd, zero, 04328    // mul* rd, rs, zero => addi rd, zero, 04329    if (MI.getOperand(1).getReg() == RISCV::X0 ||4330        MI.getOperand(2).getReg() == RISCV::X0) {4331      MI.getOperand(1).setReg(RISCV::X0);4332      MI.getOperand(2).ChangeToImmediate(0);4333      MI.setDesc(get(RISCV::ADDI));4334      return true;4335    }4336    break;4337  case RISCV::ANDI:4338    // andi rd, zero, C => addi rd, zero, 04339    if (MI.getOperand(1).getReg() == RISCV::X0) {4340      MI.getOperand(2).setImm(0);4341      MI.setDesc(get(RISCV::ADDI));4342      return true;4343    }4344    break;4345  case RISCV::SLL:4346  case RISCV::SRL:4347  case RISCV::SRA:4348    // shift rd, zero, rs => addi rd, zero, 04349    if (MI.getOperand(1).getReg() == RISCV::X0) {4350      MI.getOperand(2).ChangeToImmediate(0);4351      MI.setDesc(get(RISCV::ADDI));4352      return true;4353    }4354    // shift rd, rs, zero => addi rd, rs, 04355    if (MI.getOperand(2).getReg() == RISCV::X0) {4356      MI.getOperand(2).ChangeToImmediate(0);4357      MI.setDesc(get(RISCV::ADDI));4358      return true;4359    }4360    break;4361  case RISCV::SLLW:4362  case RISCV::SRLW:4363  case RISCV::SRAW:4364    // shiftw rd, zero, rs => addi rd, zero, 04365    if (MI.getOperand(1).getReg() == RISCV::X0) {4366      MI.getOperand(2).ChangeToImmediate(0);4367      MI.setDesc(get(RISCV::ADDI));4368      return true;4369    }4370    break;4371  case RISCV::SLLI:4372  case RISCV::SRLI:4373  case RISCV::SRAI:4374  case RISCV::SLLIW:4375  case RISCV::SRLIW:4376  case RISCV::SRAIW:4377  case RISCV::SLLI_UW:4378    // shiftimm rd, zero, N => addi rd, zero, 04379    if (MI.getOperand(1).getReg() == RISCV::X0) {4380      MI.getOperand(2).setImm(0);4381      MI.setDesc(get(RISCV::ADDI));4382      return true;4383    }4384    break;4385  case RISCV::SLTU:4386  case RISCV::ADD_UW:4387    // sltu rd, zero, zero => addi rd, zero, 04388    // add.uw rd, zero, zero => addi rd, zero, 04389    if (MI.getOperand(1).getReg() == RISCV::X0 &&4390        MI.getOperand(2).getReg() == RISCV::X0) {4391      MI.getOperand(2).ChangeToImmediate(0);4392      MI.setDesc(get(RISCV::ADDI));4393      return true;4394    }4395    // add.uw rd, zero, rs => addi rd, rs, 04396    if (MI.getOpcode() == RISCV::ADD_UW &&4397        MI.getOperand(1).getReg() == RISCV::X0) {4398      MI.removeOperand(1);4399      MI.addOperand(MachineOperand::CreateImm(0));4400      MI.setDesc(get(RISCV::ADDI));4401    }4402    break;4403  case RISCV::SLTIU:4404    // sltiu rd, zero, NZC => addi rd, zero, 14405    // sltiu rd, zero, 0 => addi rd, zero, 04406    if (MI.getOperand(1).getReg() == RISCV::X0) {4407      MI.getOperand(2).setImm(MI.getOperand(2).getImm() != 0);4408      MI.setDesc(get(RISCV::ADDI));4409      return true;4410    }4411    break;4412  case RISCV::SEXT_H:4413  case RISCV::SEXT_B:4414  case RISCV::ZEXT_H_RV32:4415  case RISCV::ZEXT_H_RV64:4416    // sext.[hb] rd, zero => addi rd, zero, 04417    // zext.h rd, zero => addi rd, zero, 04418    if (MI.getOperand(1).getReg() == RISCV::X0) {4419      MI.addOperand(MachineOperand::CreateImm(0));4420      MI.setDesc(get(RISCV::ADDI));4421      return true;4422    }4423    break;4424  case RISCV::MIN:4425  case RISCV::MINU:4426  case RISCV::MAX:4427  case RISCV::MAXU:4428    // min|max rd, rs, rs => addi rd, rs, 04429    if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {4430      MI.getOperand(2).ChangeToImmediate(0);4431      MI.setDesc(get(RISCV::ADDI));4432      return true;4433    }4434    break;4435  case RISCV::BEQ:4436  case RISCV::BNE:4437    // b{eq,ne} zero, rs, imm => b{eq,ne} rs, zero, imm4438    if (MI.getOperand(0).getReg() == RISCV::X0) {4439      MachineOperand MO0 = MI.getOperand(0);4440      MI.removeOperand(0);4441      MI.insert(MI.operands_begin() + 1, {MO0});4442    }4443    break;4444  case RISCV::BLTU:4445    // bltu zero, rs, imm => bne rs, zero, imm4446    if (MI.getOperand(0).getReg() == RISCV::X0) {4447      MachineOperand MO0 = MI.getOperand(0);4448      MI.removeOperand(0);4449      MI.insert(MI.operands_begin() + 1, {MO0});4450      MI.setDesc(get(RISCV::BNE));4451    }4452    break;4453  case RISCV::BGEU:4454    // bgeu zero, rs, imm => beq rs, zero, imm4455    if (MI.getOperand(0).getReg() == RISCV::X0) {4456      MachineOperand MO0 = MI.getOperand(0);4457      MI.removeOperand(0);4458      MI.insert(MI.operands_begin() + 1, {MO0});4459      MI.setDesc(get(RISCV::BEQ));4460    }4461    break;4462  }4463  return false;4464}4465 4466// clang-format off4467#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL)                                    \4468  RISCV::PseudoV##OP##_##LMUL##_TIED4469 4470#define CASE_WIDEOP_OPCODE_LMULS(OP)                                           \4471  CASE_WIDEOP_OPCODE_COMMON(OP, MF8):                                          \4472  case CASE_WIDEOP_OPCODE_COMMON(OP, MF4):                                     \4473  case CASE_WIDEOP_OPCODE_COMMON(OP, MF2):                                     \4474  case CASE_WIDEOP_OPCODE_COMMON(OP, M1):                                      \4475  case CASE_WIDEOP_OPCODE_COMMON(OP, M2):                                      \4476  case CASE_WIDEOP_OPCODE_COMMON(OP, M4)4477 4478#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL)                             \4479  case RISCV::PseudoV##OP##_##LMUL##_TIED:                                     \4480    NewOpc = RISCV::PseudoV##OP##_##LMUL;                                      \4481    break;4482 4483#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP)                                    \4484  CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8)                                    \4485  CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4)                                    \4486  CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2)                                    \4487  CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1)                                     \4488  CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2)                                     \4489  CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)4490 4491// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.4492#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW)                            \4493  RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED4494 4495#define CASE_FP_WIDEOP_OPCODE_LMULS(OP)                                        \4496  CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16):                                  \4497  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16):                             \4498  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32):                             \4499  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16):                              \4500  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32):                              \4501  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16):                              \4502  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32):                              \4503  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16):                              \4504  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32)                               \4505 4506#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW)                     \4507  case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED:                             \4508    NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW;                              \4509    break;4510 4511#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP)                                 \4512  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16)                            \4513  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16)                            \4514  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32)                            \4515  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16)                             \4516  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32)                             \4517  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16)                             \4518  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32)                             \4519  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16)                             \4520  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32)                             \4521 4522#define CASE_FP_WIDEOP_OPCODE_LMULS_ALT(OP)                                    \4523  CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16):                                  \4524  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16):                             \4525  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16):                              \4526  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16):                              \4527  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16)4528 4529#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(OP)                             \4530  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16)                            \4531  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16)                            \4532  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16)                             \4533  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16)                             \4534  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16)4535// clang-format on4536 4537MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,4538                                                    LiveVariables *LV,4539                                                    LiveIntervals *LIS) const {4540  MachineInstrBuilder MIB;4541  switch (MI.getOpcode()) {4542  default:4543    return nullptr;4544  case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWADD_ALT_WV):4545  case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWSUB_ALT_WV):4546  case CASE_FP_WIDEOP_OPCODE_LMULS(FWADD_WV):4547  case CASE_FP_WIDEOP_OPCODE_LMULS(FWSUB_WV): {4548    assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&4549           MI.getNumExplicitOperands() == 7 &&4550           "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");4551    // If the tail policy is undisturbed we can't convert.4552    if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &4553         1) == 0)4554      return nullptr;4555    // clang-format off4556    unsigned NewOpc;4557    switch (MI.getOpcode()) {4558    default:4559      llvm_unreachable("Unexpected opcode");4560    CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWADD_WV)4561    CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWSUB_WV)4562    CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWADD_ALT_WV)4563    CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWSUB_ALT_WV)4564    }4565    // clang-format on4566 4567    MachineBasicBlock &MBB = *MI.getParent();4568    MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))4569              .add(MI.getOperand(0))4570              .addReg(MI.getOperand(0).getReg(), RegState::Undef)4571              .add(MI.getOperand(1))4572              .add(MI.getOperand(2))4573              .add(MI.getOperand(3))4574              .add(MI.getOperand(4))4575              .add(MI.getOperand(5))4576              .add(MI.getOperand(6));4577    break;4578  }4579  case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):4580  case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):4581  case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):4582  case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {4583    // If the tail policy is undisturbed we can't convert.4584    assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&4585           MI.getNumExplicitOperands() == 6);4586    if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &4587         1) == 0)4588      return nullptr;4589 4590    // clang-format off4591    unsigned NewOpc;4592    switch (MI.getOpcode()) {4593    default:4594      llvm_unreachable("Unexpected opcode");4595    CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV)4596    CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV)4597    CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV)4598    CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV)4599    }4600    // clang-format on4601 4602    MachineBasicBlock &MBB = *MI.getParent();4603    MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))4604              .add(MI.getOperand(0))4605              .addReg(MI.getOperand(0).getReg(), RegState::Undef)4606              .add(MI.getOperand(1))4607              .add(MI.getOperand(2))4608              .add(MI.getOperand(3))4609              .add(MI.getOperand(4))4610              .add(MI.getOperand(5));4611    break;4612  }4613  }4614  MIB.copyImplicitOps(MI);4615 4616  if (LV) {4617    unsigned NumOps = MI.getNumOperands();4618    for (unsigned I = 1; I < NumOps; ++I) {4619      MachineOperand &Op = MI.getOperand(I);4620      if (Op.isReg() && Op.isKill())4621        LV->replaceKillInstruction(Op.getReg(), MI, *MIB);4622    }4623  }4624 4625  if (LIS) {4626    SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, *MIB);4627 4628    if (MI.getOperand(0).isEarlyClobber()) {4629      // Use operand 1 was tied to early-clobber def operand 0, so its live4630      // interval could have ended at an early-clobber slot. Now they are not4631      // tied we need to update it to the normal register slot.4632      LiveInterval &LI = LIS->getInterval(MI.getOperand(1).getReg());4633      LiveRange::Segment *S = LI.getSegmentContaining(Idx);4634      if (S->end == Idx.getRegSlot(true))4635        S->end = Idx.getRegSlot();4636    }4637  }4638 4639  return MIB;4640}4641 4642#undef CASE_WIDEOP_OPCODE_COMMON4643#undef CASE_WIDEOP_OPCODE_LMULS4644#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON4645#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS4646#undef CASE_FP_WIDEOP_OPCODE_COMMON4647#undef CASE_FP_WIDEOP_OPCODE_LMULS4648#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON4649#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS4650 4651void RISCVInstrInfo::mulImm(MachineFunction &MF, MachineBasicBlock &MBB,4652                            MachineBasicBlock::iterator II, const DebugLoc &DL,4653                            Register DestReg, uint32_t Amount,4654                            MachineInstr::MIFlag Flag) const {4655  MachineRegisterInfo &MRI = MF.getRegInfo();4656  if (llvm::has_single_bit<uint32_t>(Amount)) {4657    uint32_t ShiftAmount = Log2_32(Amount);4658    if (ShiftAmount == 0)4659      return;4660    BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)4661        .addReg(DestReg, RegState::Kill)4662        .addImm(ShiftAmount)4663        .setMIFlag(Flag);4664  } else if (int ShXAmount, ShiftAmount;4665             STI.hasShlAdd(3) &&4666             (ShXAmount = isShifted359(Amount, ShiftAmount)) != 0) {4667    // We can use Zba SHXADD+SLLI instructions for multiply in some cases.4668    unsigned Opc;4669    switch (ShXAmount) {4670    case 1:4671      Opc = RISCV::SH1ADD;4672      break;4673    case 2:4674      Opc = RISCV::SH2ADD;4675      break;4676    case 3:4677      Opc = RISCV::SH3ADD;4678      break;4679    default:4680      llvm_unreachable("unexpected result of isShifted359");4681    }4682    if (ShiftAmount)4683      BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)4684          .addReg(DestReg, RegState::Kill)4685          .addImm(ShiftAmount)4686          .setMIFlag(Flag);4687    BuildMI(MBB, II, DL, get(Opc), DestReg)4688        .addReg(DestReg, RegState::Kill)4689        .addReg(DestReg)4690        .setMIFlag(Flag);4691  } else if (llvm::has_single_bit<uint32_t>(Amount - 1)) {4692    Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);4693    uint32_t ShiftAmount = Log2_32(Amount - 1);4694    BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)4695        .addReg(DestReg)4696        .addImm(ShiftAmount)4697        .setMIFlag(Flag);4698    BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)4699        .addReg(ScaledRegister, RegState::Kill)4700        .addReg(DestReg, RegState::Kill)4701        .setMIFlag(Flag);4702  } else if (llvm::has_single_bit<uint32_t>(Amount + 1)) {4703    Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);4704    uint32_t ShiftAmount = Log2_32(Amount + 1);4705    BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)4706        .addReg(DestReg)4707        .addImm(ShiftAmount)4708        .setMIFlag(Flag);4709    BuildMI(MBB, II, DL, get(RISCV::SUB), DestReg)4710        .addReg(ScaledRegister, RegState::Kill)4711        .addReg(DestReg, RegState::Kill)4712        .setMIFlag(Flag);4713  } else if (STI.hasStdExtZmmul()) {4714    Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass);4715    movImm(MBB, II, DL, N, Amount, Flag);4716    BuildMI(MBB, II, DL, get(RISCV::MUL), DestReg)4717        .addReg(DestReg, RegState::Kill)4718        .addReg(N, RegState::Kill)4719        .setMIFlag(Flag);4720  } else {4721    Register Acc;4722    uint32_t PrevShiftAmount = 0;4723    for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {4724      if (Amount & (1U << ShiftAmount)) {4725        if (ShiftAmount)4726          BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)4727              .addReg(DestReg, RegState::Kill)4728              .addImm(ShiftAmount - PrevShiftAmount)4729              .setMIFlag(Flag);4730        if (Amount >> (ShiftAmount + 1)) {4731          // If we don't have an accmulator yet, create it and copy DestReg.4732          if (!Acc) {4733            Acc = MRI.createVirtualRegister(&RISCV::GPRRegClass);4734            BuildMI(MBB, II, DL, get(TargetOpcode::COPY), Acc)4735                .addReg(DestReg)4736                .setMIFlag(Flag);4737          } else {4738            BuildMI(MBB, II, DL, get(RISCV::ADD), Acc)4739                .addReg(Acc, RegState::Kill)4740                .addReg(DestReg)4741                .setMIFlag(Flag);4742          }4743        }4744        PrevShiftAmount = ShiftAmount;4745      }4746    }4747    assert(Acc && "Expected valid accumulator");4748    BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)4749        .addReg(DestReg, RegState::Kill)4750        .addReg(Acc, RegState::Kill)4751        .setMIFlag(Flag);4752  }4753}4754 4755ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>4756RISCVInstrInfo::getSerializableMachineMemOperandTargetFlags() const {4757  static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =4758      {{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},4759       {MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};4760  return ArrayRef(TargetFlags);4761}4762 4763unsigned RISCVInstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {4764  return OptLevel >= CodeGenOptLevel::Aggressive4765             ? STI.getTailDupAggressiveThreshold()4766             : 2;4767}4768 4769bool RISCV::isRVVSpill(const MachineInstr &MI) {4770  // RVV lacks any support for immediate addressing for stack addresses, so be4771  // conservative.4772  unsigned Opcode = MI.getOpcode();4773  if (!RISCVVPseudosTable::getPseudoInfo(Opcode) &&4774      !getLMULForRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode))4775    return false;4776  return true;4777}4778 4779std::optional<std::pair<unsigned, unsigned>>4780RISCV::isRVVSpillForZvlsseg(unsigned Opcode) {4781  switch (Opcode) {4782  default:4783    return std::nullopt;4784  case RISCV::PseudoVSPILL2_M1:4785  case RISCV::PseudoVRELOAD2_M1:4786    return std::make_pair(2u, 1u);4787  case RISCV::PseudoVSPILL2_M2:4788  case RISCV::PseudoVRELOAD2_M2:4789    return std::make_pair(2u, 2u);4790  case RISCV::PseudoVSPILL2_M4:4791  case RISCV::PseudoVRELOAD2_M4:4792    return std::make_pair(2u, 4u);4793  case RISCV::PseudoVSPILL3_M1:4794  case RISCV::PseudoVRELOAD3_M1:4795    return std::make_pair(3u, 1u);4796  case RISCV::PseudoVSPILL3_M2:4797  case RISCV::PseudoVRELOAD3_M2:4798    return std::make_pair(3u, 2u);4799  case RISCV::PseudoVSPILL4_M1:4800  case RISCV::PseudoVRELOAD4_M1:4801    return std::make_pair(4u, 1u);4802  case RISCV::PseudoVSPILL4_M2:4803  case RISCV::PseudoVRELOAD4_M2:4804    return std::make_pair(4u, 2u);4805  case RISCV::PseudoVSPILL5_M1:4806  case RISCV::PseudoVRELOAD5_M1:4807    return std::make_pair(5u, 1u);4808  case RISCV::PseudoVSPILL6_M1:4809  case RISCV::PseudoVRELOAD6_M1:4810    return std::make_pair(6u, 1u);4811  case RISCV::PseudoVSPILL7_M1:4812  case RISCV::PseudoVRELOAD7_M1:4813    return std::make_pair(7u, 1u);4814  case RISCV::PseudoVSPILL8_M1:4815  case RISCV::PseudoVRELOAD8_M1:4816    return std::make_pair(8u, 1u);4817  }4818}4819 4820bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {4821  int16_t MI1FrmOpIdx =4822      RISCV::getNamedOperandIdx(MI1.getOpcode(), RISCV::OpName::frm);4823  int16_t MI2FrmOpIdx =4824      RISCV::getNamedOperandIdx(MI2.getOpcode(), RISCV::OpName::frm);4825  if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)4826    return false;4827  MachineOperand FrmOp1 = MI1.getOperand(MI1FrmOpIdx);4828  MachineOperand FrmOp2 = MI2.getOperand(MI2FrmOpIdx);4829  return FrmOp1.getImm() == FrmOp2.getImm();4830}4831 4832std::optional<unsigned>4833RISCV::getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW) {4834  switch (Opcode) {4835  default:4836    return std::nullopt;4837 4838  // 11.6. Vector Single-Width Shift Instructions4839  case RISCV::VSLL_VX:4840  case RISCV::VSRL_VX:4841  case RISCV::VSRA_VX:4842  // 12.4. Vector Single-Width Scaling Shift Instructions4843  case RISCV::VSSRL_VX:4844  case RISCV::VSSRA_VX:4845  // Zvbb4846  case RISCV::VROL_VX:4847  case RISCV::VROR_VX:4848    // Only the low lg2(SEW) bits of the shift-amount value are used.4849    return Log2SEW;4850 4851  // 11.7 Vector Narrowing Integer Right Shift Instructions4852  case RISCV::VNSRL_WX:4853  case RISCV::VNSRA_WX:4854  // 12.5. Vector Narrowing Fixed-Point Clip Instructions4855  case RISCV::VNCLIPU_WX:4856  case RISCV::VNCLIP_WX:4857  // Zvbb4858  case RISCV::VWSLL_VX:4859    // Only the low lg2(2*SEW) bits of the shift-amount value are used.4860    return Log2SEW + 1;4861 4862  // 11.1. Vector Single-Width Integer Add and Subtract4863  case RISCV::VADD_VX:4864  case RISCV::VSUB_VX:4865  case RISCV::VRSUB_VX:4866  // 11.2. Vector Widening Integer Add/Subtract4867  case RISCV::VWADDU_VX:4868  case RISCV::VWSUBU_VX:4869  case RISCV::VWADD_VX:4870  case RISCV::VWSUB_VX:4871  case RISCV::VWADDU_WX:4872  case RISCV::VWSUBU_WX:4873  case RISCV::VWADD_WX:4874  case RISCV::VWSUB_WX:4875  // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions4876  case RISCV::VADC_VXM:4877  case RISCV::VADC_VIM:4878  case RISCV::VMADC_VXM:4879  case RISCV::VMADC_VIM:4880  case RISCV::VMADC_VX:4881  case RISCV::VSBC_VXM:4882  case RISCV::VMSBC_VXM:4883  case RISCV::VMSBC_VX:4884  // 11.5 Vector Bitwise Logical Instructions4885  case RISCV::VAND_VX:4886  case RISCV::VOR_VX:4887  case RISCV::VXOR_VX:4888  // 11.8. Vector Integer Compare Instructions4889  case RISCV::VMSEQ_VX:4890  case RISCV::VMSNE_VX:4891  case RISCV::VMSLTU_VX:4892  case RISCV::VMSLT_VX:4893  case RISCV::VMSLEU_VX:4894  case RISCV::VMSLE_VX:4895  case RISCV::VMSGTU_VX:4896  case RISCV::VMSGT_VX:4897  // 11.9. Vector Integer Min/Max Instructions4898  case RISCV::VMINU_VX:4899  case RISCV::VMIN_VX:4900  case RISCV::VMAXU_VX:4901  case RISCV::VMAX_VX:4902  // 11.10. Vector Single-Width Integer Multiply Instructions4903  case RISCV::VMUL_VX:4904  case RISCV::VMULH_VX:4905  case RISCV::VMULHU_VX:4906  case RISCV::VMULHSU_VX:4907  // 11.11. Vector Integer Divide Instructions4908  case RISCV::VDIVU_VX:4909  case RISCV::VDIV_VX:4910  case RISCV::VREMU_VX:4911  case RISCV::VREM_VX:4912  // 11.12. Vector Widening Integer Multiply Instructions4913  case RISCV::VWMUL_VX:4914  case RISCV::VWMULU_VX:4915  case RISCV::VWMULSU_VX:4916  // 11.13. Vector Single-Width Integer Multiply-Add Instructions4917  case RISCV::VMACC_VX:4918  case RISCV::VNMSAC_VX:4919  case RISCV::VMADD_VX:4920  case RISCV::VNMSUB_VX:4921  // 11.14. Vector Widening Integer Multiply-Add Instructions4922  case RISCV::VWMACCU_VX:4923  case RISCV::VWMACC_VX:4924  case RISCV::VWMACCSU_VX:4925  case RISCV::VWMACCUS_VX:4926  // 11.15. Vector Integer Merge Instructions4927  case RISCV::VMERGE_VXM:4928  // 11.16. Vector Integer Move Instructions4929  case RISCV::VMV_V_X:4930  // 12.1. Vector Single-Width Saturating Add and Subtract4931  case RISCV::VSADDU_VX:4932  case RISCV::VSADD_VX:4933  case RISCV::VSSUBU_VX:4934  case RISCV::VSSUB_VX:4935  // 12.2. Vector Single-Width Averaging Add and Subtract4936  case RISCV::VAADDU_VX:4937  case RISCV::VAADD_VX:4938  case RISCV::VASUBU_VX:4939  case RISCV::VASUB_VX:4940  // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation4941  case RISCV::VSMUL_VX:4942  // 16.1. Integer Scalar Move Instructions4943  case RISCV::VMV_S_X:4944  // Zvbb4945  case RISCV::VANDN_VX:4946    return 1U << Log2SEW;4947  }4948}4949 4950unsigned RISCV::getRVVMCOpcode(unsigned RVVPseudoOpcode) {4951  const RISCVVPseudosTable::PseudoInfo *RVV =4952      RISCVVPseudosTable::getPseudoInfo(RVVPseudoOpcode);4953  if (!RVV)4954    return 0;4955  return RVV->BaseInstr;4956}4957 4958unsigned RISCV::getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW) {4959  unsigned DestEEW =4960      (Desc.TSFlags & RISCVII::DestEEWMask) >> RISCVII::DestEEWShift;4961  // EEW = 14962  if (DestEEW == 0)4963    return 0;4964  // EEW = SEW * n4965  unsigned Scaled = Log2SEW + (DestEEW - 1);4966  assert(Scaled >= 3 && Scaled <= 6);4967  return Scaled;4968}4969 4970static std::optional<int64_t> getEffectiveImm(const MachineOperand &MO) {4971  assert(MO.isImm() || MO.getReg().isVirtual());4972  if (MO.isImm())4973    return MO.getImm();4974  const MachineInstr *Def =4975      MO.getParent()->getMF()->getRegInfo().getVRegDef(MO.getReg());4976  int64_t Imm;4977  if (isLoadImm(Def, Imm))4978    return Imm;4979  return std::nullopt;4980}4981 4982/// Given two VL operands, do we know that LHS <= RHS? Must be used in SSA form.4983bool RISCV::isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS) {4984  assert((LHS.isImm() || LHS.getParent()->getMF()->getRegInfo().isSSA()) &&4985         (RHS.isImm() || RHS.getParent()->getMF()->getRegInfo().isSSA()));4986  if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&4987      LHS.getReg() == RHS.getReg())4988    return true;4989  if (RHS.isImm() && RHS.getImm() == RISCV::VLMaxSentinel)4990    return true;4991  if (LHS.isImm() && LHS.getImm() == 0)4992    return true;4993  if (LHS.isImm() && LHS.getImm() == RISCV::VLMaxSentinel)4994    return false;4995  std::optional<int64_t> LHSImm = getEffectiveImm(LHS),4996                         RHSImm = getEffectiveImm(RHS);4997  if (!LHSImm || !RHSImm)4998    return false;4999  return LHSImm <= RHSImm;5000}5001 5002namespace {5003class RISCVPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {5004  const MachineInstr *LHS;5005  const MachineInstr *RHS;5006  SmallVector<MachineOperand, 3> Cond;5007 5008public:5009  RISCVPipelinerLoopInfo(const MachineInstr *LHS, const MachineInstr *RHS,5010                         const SmallVectorImpl<MachineOperand> &Cond)5011      : LHS(LHS), RHS(RHS), Cond(Cond.begin(), Cond.end()) {}5012 5013  bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {5014    // Make the instructions for loop control be placed in stage 0.5015    // The predecessors of LHS/RHS are considered by the caller.5016    if (LHS && MI == LHS)5017      return true;5018    if (RHS && MI == RHS)5019      return true;5020    return false;5021  }5022 5023  std::optional<bool> createTripCountGreaterCondition(5024      int TC, MachineBasicBlock &MBB,5025      SmallVectorImpl<MachineOperand> &CondParam) override {5026    // A branch instruction will be inserted as "if (Cond) goto epilogue".5027    // Cond is normalized for such use.5028    // The predecessors of the branch are assumed to have already been inserted.5029    CondParam = Cond;5030    return {};5031  }5032 5033  void setPreheader(MachineBasicBlock *NewPreheader) override {}5034 5035  void adjustTripCount(int TripCountAdjust) override {}5036};5037} // namespace5038 5039std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>5040RISCVInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {5041  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;5042  SmallVector<MachineOperand, 4> Cond;5043  if (analyzeBranch(*LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))5044    return nullptr;5045 5046  // Infinite loops are not supported5047  if (TBB == LoopBB && FBB == LoopBB)5048    return nullptr;5049 5050  // Must be conditional branch5051  if (FBB == nullptr)5052    return nullptr;5053 5054  assert((TBB == LoopBB || FBB == LoopBB) &&5055         "The Loop must be a single-basic-block loop");5056 5057  // Normalization for createTripCountGreaterCondition()5058  if (TBB == LoopBB)5059    reverseBranchCondition(Cond);5060 5061  const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();5062  auto FindRegDef = [&MRI](MachineOperand &Op) -> const MachineInstr * {5063    if (!Op.isReg())5064      return nullptr;5065    Register Reg = Op.getReg();5066    if (!Reg.isVirtual())5067      return nullptr;5068    return MRI.getVRegDef(Reg);5069  };5070 5071  const MachineInstr *LHS = FindRegDef(Cond[1]);5072  const MachineInstr *RHS = FindRegDef(Cond[2]);5073  if (LHS && LHS->isPHI())5074    return nullptr;5075  if (RHS && RHS->isPHI())5076    return nullptr;5077 5078  return std::make_unique<RISCVPipelinerLoopInfo>(LHS, RHS, Cond);5079}5080 5081// FIXME: We should remove this if we have a default generic scheduling model.5082bool RISCVInstrInfo::isHighLatencyDef(int Opc) const {5083  unsigned RVVMCOpcode = RISCV::getRVVMCOpcode(Opc);5084  Opc = RVVMCOpcode ? RVVMCOpcode : Opc;5085  switch (Opc) {5086  default:5087    return false;5088  // Integer div/rem.5089  case RISCV::DIV:5090  case RISCV::DIVW:5091  case RISCV::DIVU:5092  case RISCV::DIVUW:5093  case RISCV::REM:5094  case RISCV::REMW:5095  case RISCV::REMU:5096  case RISCV::REMUW:5097  // Floating-point div/sqrt.5098  case RISCV::FDIV_H:5099  case RISCV::FDIV_S:5100  case RISCV::FDIV_D:5101  case RISCV::FDIV_H_INX:5102  case RISCV::FDIV_S_INX:5103  case RISCV::FDIV_D_INX:5104  case RISCV::FDIV_D_IN32X:5105  case RISCV::FSQRT_H:5106  case RISCV::FSQRT_S:5107  case RISCV::FSQRT_D:5108  case RISCV::FSQRT_H_INX:5109  case RISCV::FSQRT_S_INX:5110  case RISCV::FSQRT_D_INX:5111  case RISCV::FSQRT_D_IN32X:5112  // Vector integer div/rem5113  case RISCV::VDIV_VV:5114  case RISCV::VDIV_VX:5115  case RISCV::VDIVU_VV:5116  case RISCV::VDIVU_VX:5117  case RISCV::VREM_VV:5118  case RISCV::VREM_VX:5119  case RISCV::VREMU_VV:5120  case RISCV::VREMU_VX:5121  // Vector floating-point div/sqrt.5122  case RISCV::VFDIV_VV:5123  case RISCV::VFDIV_VF:5124  case RISCV::VFRDIV_VF:5125  case RISCV::VFSQRT_V:5126  case RISCV::VFRSQRT7_V:5127    return true;5128  }5129}5130