brintos / llvm-project-archived public Read only

1//===-- SystemZInstrInfo.cpp - SystemZ instruction information ------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains the SystemZ implementation of the TargetInstrInfo class.10//11//===----------------------------------------------------------------------===//12 13#include "SystemZInstrInfo.h"14#include "MCTargetDesc/SystemZMCTargetDesc.h"15#include "SystemZ.h"16#include "SystemZInstrBuilder.h"17#include "SystemZSubtarget.h"18#include "llvm/ADT/Statistic.h"19#include "llvm/CodeGen/LiveInterval.h"20#include "llvm/CodeGen/LiveIntervals.h"21#include "llvm/CodeGen/LiveRegUnits.h"22#include "llvm/CodeGen/LiveVariables.h"23#include "llvm/CodeGen/MachineBasicBlock.h"24#include "llvm/CodeGen/MachineFrameInfo.h"25#include "llvm/CodeGen/MachineFunction.h"26#include "llvm/CodeGen/MachineInstr.h"27#include "llvm/CodeGen/MachineMemOperand.h"28#include "llvm/CodeGen/MachineOperand.h"29#include "llvm/CodeGen/MachineRegisterInfo.h"30#include "llvm/CodeGen/SlotIndexes.h"31#include "llvm/CodeGen/StackMaps.h"32#include "llvm/CodeGen/TargetInstrInfo.h"33#include "llvm/CodeGen/TargetOpcodes.h"34#include "llvm/CodeGen/TargetSubtargetInfo.h"35#include "llvm/CodeGen/VirtRegMap.h"36#include "llvm/MC/MCInstrDesc.h"37#include "llvm/MC/MCRegisterInfo.h"38#include "llvm/Support/BranchProbability.h"39#include "llvm/Support/ErrorHandling.h"40#include "llvm/Support/MathExtras.h"41#include "llvm/Target/TargetMachine.h"42#include <cassert>43#include <cstdint>44#include <iterator>45 46using namespace llvm;47 48#define GET_INSTRINFO_CTOR_DTOR49#define GET_INSTRMAP_INFO50#include "SystemZGenInstrInfo.inc"51 52#define DEBUG_TYPE "systemz-II"53 54// Return a mask with Count low bits set.55static uint64_t allOnes(unsigned int Count) {56  return Count == 0 ? 0 : (uint64_t(1) << (Count - 1) << 1) - 1;57}58 59// Pin the vtable to this file.60void SystemZInstrInfo::anchor() {}61 62SystemZInstrInfo::SystemZInstrInfo(const SystemZSubtarget &sti)63    : SystemZGenInstrInfo(sti, RI, -1, -1),64      RI(sti.getSpecialRegisters()->getReturnFunctionAddressRegister(),65         sti.getHwMode()),66      STI(sti) {}67 68// MI is a 128-bit load or store.  Split it into two 64-bit loads or stores,69// each having the opcode given by NewOpcode.70void SystemZInstrInfo::splitMove(MachineBasicBlock::iterator MI,71                                 unsigned NewOpcode) const {72  MachineBasicBlock *MBB = MI->getParent();73  MachineFunction &MF = *MBB->getParent();74 75  // Get two load or store instructions.  Use the original instruction for76  // one of them and create a clone for the other.77  MachineInstr *HighPartMI = MF.CloneMachineInstr(&*MI);78  MachineInstr *LowPartMI = &*MI;79  MBB->insert(LowPartMI, HighPartMI);80 81  // Set up the two 64-bit registers and remember super reg and its flags.82  MachineOperand &HighRegOp = HighPartMI->getOperand(0);83  MachineOperand &LowRegOp = LowPartMI->getOperand(0);84  Register Reg128 = LowRegOp.getReg();85  unsigned Reg128Killed = getKillRegState(LowRegOp.isKill());86  unsigned Reg128Undef  = getUndefRegState(LowRegOp.isUndef());87  HighRegOp.setReg(RI.getSubReg(HighRegOp.getReg(), SystemZ::subreg_h64));88  LowRegOp.setReg(RI.getSubReg(LowRegOp.getReg(), SystemZ::subreg_l64));89 90  // The address in the first (high) instruction is already correct.91  // Adjust the offset in the second (low) instruction.92  MachineOperand &HighOffsetOp = HighPartMI->getOperand(2);93  MachineOperand &LowOffsetOp = LowPartMI->getOperand(2);94  LowOffsetOp.setImm(LowOffsetOp.getImm() + 8);95 96  // Set the opcodes.97  unsigned HighOpcode = getOpcodeForOffset(NewOpcode, HighOffsetOp.getImm());98  unsigned LowOpcode = getOpcodeForOffset(NewOpcode, LowOffsetOp.getImm());99  assert(HighOpcode && LowOpcode && "Both offsets should be in range");100  HighPartMI->setDesc(get(HighOpcode));101  LowPartMI->setDesc(get(LowOpcode));102 103  MachineInstr *FirstMI = HighPartMI;104  if (MI->mayStore()) {105    FirstMI->getOperand(0).setIsKill(false);106    // Add implicit uses of the super register in case one of the subregs is107    // undefined. We could track liveness and skip storing an undefined108    // subreg, but this is hopefully rare (discovered with llvm-stress).109    // If Reg128 was killed, set kill flag on MI.110    unsigned Reg128UndefImpl = (Reg128Undef | RegState::Implicit);111    MachineInstrBuilder(MF, HighPartMI).addReg(Reg128, Reg128UndefImpl);112    MachineInstrBuilder(MF, LowPartMI).addReg(Reg128, (Reg128UndefImpl | Reg128Killed));113  } else {114    // If HighPartMI clobbers any of the address registers, it needs to come115    // after LowPartMI.116    auto overlapsAddressReg = [&](Register Reg) -> bool {117      return RI.regsOverlap(Reg, MI->getOperand(1).getReg()) ||118             RI.regsOverlap(Reg, MI->getOperand(3).getReg());119    };120    if (overlapsAddressReg(HighRegOp.getReg())) {121      assert(!overlapsAddressReg(LowRegOp.getReg()) &&122             "Both loads clobber address!");123      MBB->splice(HighPartMI, MBB, LowPartMI);124      FirstMI = LowPartMI;125    }126  }127 128  // Clear the kill flags on the address registers in the first instruction.129  FirstMI->getOperand(1).setIsKill(false);130  FirstMI->getOperand(3).setIsKill(false);131}132 133// Split ADJDYNALLOC instruction MI.134void SystemZInstrInfo::splitAdjDynAlloc(MachineBasicBlock::iterator MI) const {135  MachineBasicBlock *MBB = MI->getParent();136  MachineFunction &MF = *MBB->getParent();137  MachineFrameInfo &MFFrame = MF.getFrameInfo();138  MachineOperand &OffsetMO = MI->getOperand(2);139  SystemZCallingConventionRegisters *Regs = STI.getSpecialRegisters();140 141  uint64_t Offset = (MFFrame.getMaxCallFrameSize() +142                     Regs->getCallFrameSize() +143                     Regs->getStackPointerBias() +144                     OffsetMO.getImm());145  unsigned NewOpcode = getOpcodeForOffset(SystemZ::LA, Offset);146  assert(NewOpcode && "No support for huge argument lists yet");147  MI->setDesc(get(NewOpcode));148  OffsetMO.setImm(Offset);149}150 151// MI is an RI-style pseudo instruction.  Replace it with LowOpcode152// if the first operand is a low GR32 and HighOpcode if the first operand153// is a high GR32.  ConvertHigh is true if LowOpcode takes a signed operand154// and HighOpcode takes an unsigned 32-bit operand.  In those cases,155// MI has the same kind of operand as LowOpcode, so needs to be converted156// if HighOpcode is used.157void SystemZInstrInfo::expandRIPseudo(MachineInstr &MI, unsigned LowOpcode,158                                      unsigned HighOpcode,159                                      bool ConvertHigh) const {160  Register Reg = MI.getOperand(0).getReg();161  bool IsHigh = SystemZ::isHighReg(Reg);162  MI.setDesc(get(IsHigh ? HighOpcode : LowOpcode));163  if (IsHigh && ConvertHigh)164    MI.getOperand(1).setImm(uint32_t(MI.getOperand(1).getImm()));165}166 167// MI is a three-operand RIE-style pseudo instruction.  Replace it with168// LowOpcodeK if the registers are both low GR32s, otherwise use a move169// followed by HighOpcode or LowOpcode, depending on whether the target170// is a high or low GR32.171void SystemZInstrInfo::expandRIEPseudo(MachineInstr &MI, unsigned LowOpcode,172                                       unsigned LowOpcodeK,173                                       unsigned HighOpcode) const {174  Register DestReg = MI.getOperand(0).getReg();175  Register SrcReg = MI.getOperand(1).getReg();176  bool DestIsHigh = SystemZ::isHighReg(DestReg);177  bool SrcIsHigh = SystemZ::isHighReg(SrcReg);178  if (!DestIsHigh && !SrcIsHigh)179    MI.setDesc(get(LowOpcodeK));180  else {181    if (DestReg != SrcReg) {182      emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, SrcReg,183                    SystemZ::LR, 32, MI.getOperand(1).isKill(),184                    MI.getOperand(1).isUndef());185      MI.getOperand(1).setReg(DestReg);186    }187    MI.setDesc(get(DestIsHigh ? HighOpcode : LowOpcode));188    MI.tieOperands(0, 1);189  }190}191 192// MI is an RXY-style pseudo instruction.  Replace it with LowOpcode193// if the first operand is a low GR32 and HighOpcode if the first operand194// is a high GR32.195void SystemZInstrInfo::expandRXYPseudo(MachineInstr &MI, unsigned LowOpcode,196                                       unsigned HighOpcode) const {197  Register Reg = MI.getOperand(0).getReg();198  unsigned Opcode = getOpcodeForOffset(199      SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode,200      MI.getOperand(2).getImm());201  MI.setDesc(get(Opcode));202}203 204// MI is a load-on-condition pseudo instruction with a single register205// (source or destination) operand.  Replace it with LowOpcode if the206// register is a low GR32 and HighOpcode if the register is a high GR32.207void SystemZInstrInfo::expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode,208                                       unsigned HighOpcode) const {209  Register Reg = MI.getOperand(0).getReg();210  unsigned Opcode = SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode;211  MI.setDesc(get(Opcode));212}213 214// MI is an RR-style pseudo instruction that zero-extends the low Size bits215// of one GRX32 into another.  Replace it with LowOpcode if both operands216// are low registers, otherwise use RISB[LH]G.217void SystemZInstrInfo::expandZExtPseudo(MachineInstr &MI, unsigned LowOpcode,218                                        unsigned Size) const {219  MachineInstrBuilder MIB =220    emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(),221               MI.getOperand(0).getReg(), MI.getOperand(1).getReg(), LowOpcode,222               Size, MI.getOperand(1).isKill(), MI.getOperand(1).isUndef());223 224  // Keep the remaining operands as-is.225  for (const MachineOperand &MO : llvm::drop_begin(MI.operands(), 2))226    MIB.add(MO);227 228  MI.eraseFromParent();229}230 231void SystemZInstrInfo::expandLoadStackGuard(MachineInstr *MI) const {232  MachineBasicBlock *MBB = MI->getParent();233  MachineFunction &MF = *MBB->getParent();234  const Register Reg64 = MI->getOperand(0).getReg();235  const Register Reg32 = RI.getSubReg(Reg64, SystemZ::subreg_l32);236 237  // EAR can only load the low subregister so us a shift for %a0 to produce238  // the GR containing %a0 and %a1.239 240  // ear <reg>, %a0241  BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32)242    .addReg(SystemZ::A0)243    .addReg(Reg64, RegState::ImplicitDefine);244 245  // sllg <reg>, <reg>, 32246  BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::SLLG), Reg64)247    .addReg(Reg64)248    .addReg(0)249    .addImm(32);250 251  // ear <reg>, %a1252  BuildMI(*MBB, MI, MI->getDebugLoc(), get(SystemZ::EAR), Reg32)253    .addReg(SystemZ::A1);254 255  // lg <reg>, 40(<reg>)256  MI->setDesc(get(SystemZ::LG));257  MachineInstrBuilder(MF, MI).addReg(Reg64).addImm(40).addReg(0);258}259 260// Emit a zero-extending move from 32-bit GPR SrcReg to 32-bit GPR261// DestReg before MBBI in MBB.  Use LowLowOpcode when both DestReg and SrcReg262// are low registers, otherwise use RISB[LH]G.  Size is the number of bits263// taken from the low end of SrcReg (8 for LLCR, 16 for LLHR and 32 for LR).264// KillSrc is true if this move is the last use of SrcReg.265MachineInstrBuilder266SystemZInstrInfo::emitGRX32Move(MachineBasicBlock &MBB,267                                MachineBasicBlock::iterator MBBI,268                                const DebugLoc &DL, unsigned DestReg,269                                unsigned SrcReg, unsigned LowLowOpcode,270                                unsigned Size, bool KillSrc,271                                bool UndefSrc) const {272  unsigned Opcode;273  bool DestIsHigh = SystemZ::isHighReg(DestReg);274  bool SrcIsHigh = SystemZ::isHighReg(SrcReg);275  if (DestIsHigh && SrcIsHigh)276    Opcode = SystemZ::RISBHH;277  else if (DestIsHigh && !SrcIsHigh)278    Opcode = SystemZ::RISBHL;279  else if (!DestIsHigh && SrcIsHigh)280    Opcode = SystemZ::RISBLH;281  else {282    return BuildMI(MBB, MBBI, DL, get(LowLowOpcode), DestReg)283      .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc));284  }285  unsigned Rotate = (DestIsHigh != SrcIsHigh ? 32 : 0);286  return BuildMI(MBB, MBBI, DL, get(Opcode), DestReg)287    .addReg(DestReg, RegState::Undef)288    .addReg(SrcReg, getKillRegState(KillSrc) | getUndefRegState(UndefSrc))289    .addImm(32 - Size).addImm(128 + 31).addImm(Rotate);290}291 292MachineInstr *SystemZInstrInfo::commuteInstructionImpl(MachineInstr &MI,293                                                       bool NewMI,294                                                       unsigned OpIdx1,295                                                       unsigned OpIdx2) const {296  auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {297    if (NewMI)298      return *MI.getParent()->getParent()->CloneMachineInstr(&MI);299    return MI;300  };301 302  switch (MI.getOpcode()) {303  case SystemZ::SELRMux:304  case SystemZ::SELFHR:305  case SystemZ::SELR:306  case SystemZ::SELGR:307  case SystemZ::LOCRMux:308  case SystemZ::LOCFHR:309  case SystemZ::LOCR:310  case SystemZ::LOCGR: {311    auto &WorkingMI = cloneIfNew(MI);312    // Invert condition.313    unsigned CCValid = WorkingMI.getOperand(3).getImm();314    unsigned CCMask = WorkingMI.getOperand(4).getImm();315    WorkingMI.getOperand(4).setImm(CCMask ^ CCValid);316    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,317                                                   OpIdx1, OpIdx2);318  }319  default:320    return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);321  }322}323 324// If MI is a simple load or store for a frame object, return the register325// it loads or stores and set FrameIndex to the index of the frame object.326// Return 0 otherwise.327//328// Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores.329static int isSimpleMove(const MachineInstr &MI, int &FrameIndex,330                        unsigned Flag) {331  const MCInstrDesc &MCID = MI.getDesc();332  if ((MCID.TSFlags & Flag) && MI.getOperand(1).isFI() &&333      MI.getOperand(2).getImm() == 0 && MI.getOperand(3).getReg() == 0) {334    FrameIndex = MI.getOperand(1).getIndex();335    return MI.getOperand(0).getReg();336  }337  return 0;338}339 340Register SystemZInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,341                                               int &FrameIndex) const {342  return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXLoad);343}344 345Register SystemZInstrInfo::isStoreToStackSlot(const MachineInstr &MI,346                                              int &FrameIndex) const {347  return isSimpleMove(MI, FrameIndex, SystemZII::SimpleBDXStore);348}349 350Register SystemZInstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,351                                                     int &FrameIndex) const {352  // if this is not a simple load from memory, it's not a load from stack slot353  // either.354  const MCInstrDesc &MCID = MI.getDesc();355  if (!(MCID.TSFlags & SystemZII::SimpleBDXLoad))356    return 0;357 358  // This version of isLoadFromStackSlot should only be used post frame-index359  // elimination.360  assert(!MI.getOperand(1).isFI());361 362  // Now attempt to derive frame index from MachineMemOperands.363  SmallVector<const MachineMemOperand *, 1> Accesses;364  if (hasLoadFromStackSlot(MI, Accesses)) {365    FrameIndex =366        cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())367            ->getFrameIndex();368    return MI.getOperand(0).getReg();369  }370  return 0;371}372 373Register SystemZInstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,374                                                    int &FrameIndex) const {375  // if this is not a simple store to memory, it's not a store to stack slot376  // either.377  const MCInstrDesc &MCID = MI.getDesc();378  if (!(MCID.TSFlags & SystemZII::SimpleBDXStore))379    return 0;380 381  // This version of isStoreToStackSlot should only be used post frame-index382  // elimination.383  assert(!MI.getOperand(1).isFI());384 385  // Now attempt to derive frame index from MachineMemOperands.386  SmallVector<const MachineMemOperand *, 1> Accesses;387  if (hasStoreToStackSlot(MI, Accesses)) {388    FrameIndex =389        cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())390            ->getFrameIndex();391    return MI.getOperand(0).getReg();392  }393  return 0;394}395 396bool SystemZInstrInfo::isStackSlotCopy(const MachineInstr &MI,397                                       int &DestFrameIndex,398                                       int &SrcFrameIndex) const {399  // Check for MVC 0(Length,FI1),0(FI2)400  const MachineFrameInfo &MFI = MI.getParent()->getParent()->getFrameInfo();401  if (MI.getOpcode() != SystemZ::MVC || !MI.getOperand(0).isFI() ||402      MI.getOperand(1).getImm() != 0 || !MI.getOperand(3).isFI() ||403      MI.getOperand(4).getImm() != 0)404    return false;405 406  // Check that Length covers the full slots.407  int64_t Length = MI.getOperand(2).getImm();408  unsigned FI1 = MI.getOperand(0).getIndex();409  unsigned FI2 = MI.getOperand(3).getIndex();410  if (MFI.getObjectSize(FI1) != Length ||411      MFI.getObjectSize(FI2) != Length)412    return false;413 414  DestFrameIndex = FI1;415  SrcFrameIndex = FI2;416  return true;417}418 419bool SystemZInstrInfo::analyzeBranch(MachineBasicBlock &MBB,420                                     MachineBasicBlock *&TBB,421                                     MachineBasicBlock *&FBB,422                                     SmallVectorImpl<MachineOperand> &Cond,423                                     bool AllowModify) const {424  // Most of the code and comments here are boilerplate.425 426  // Start from the bottom of the block and work up, examining the427  // terminator instructions.428  MachineBasicBlock::iterator I = MBB.end();429  while (I != MBB.begin()) {430    --I;431    if (I->isDebugInstr())432      continue;433 434    // Working from the bottom, when we see a non-terminator instruction, we're435    // done.436    if (!isUnpredicatedTerminator(*I))437      break;438 439    // A terminator that isn't a branch can't easily be handled by this440    // analysis.441    if (!I->isBranch())442      return true;443 444    // Can't handle indirect branches.445    SystemZII::Branch Branch(getBranchInfo(*I));446    if (!Branch.hasMBBTarget())447      return true;448 449    // Punt on compound branches.450    if (Branch.Type != SystemZII::BranchNormal)451      return true;452 453    if (Branch.CCMask == SystemZ::CCMASK_ANY) {454      // Handle unconditional branches.455      if (!AllowModify) {456        TBB = Branch.getMBBTarget();457        continue;458      }459 460      // If the block has any instructions after a JMP, delete them.461      MBB.erase(std::next(I), MBB.end());462 463      Cond.clear();464      FBB = nullptr;465 466      // Delete the JMP if it's equivalent to a fall-through.467      if (MBB.isLayoutSuccessor(Branch.getMBBTarget())) {468        TBB = nullptr;469        I->eraseFromParent();470        I = MBB.end();471        continue;472      }473 474      // TBB is used to indicate the unconditinal destination.475      TBB = Branch.getMBBTarget();476      continue;477    }478 479    // Working from the bottom, handle the first conditional branch.480    if (Cond.empty()) {481      // FIXME: add X86-style branch swap482      FBB = TBB;483      TBB = Branch.getMBBTarget();484      Cond.push_back(MachineOperand::CreateImm(Branch.CCValid));485      Cond.push_back(MachineOperand::CreateImm(Branch.CCMask));486      continue;487    }488 489    // Handle subsequent conditional branches.490    assert(Cond.size() == 2 && TBB && "Should have seen a conditional branch");491 492    // Only handle the case where all conditional branches branch to the same493    // destination.494    if (TBB != Branch.getMBBTarget())495      return true;496 497    // If the conditions are the same, we can leave them alone.498    unsigned OldCCValid = Cond[0].getImm();499    unsigned OldCCMask = Cond[1].getImm();500    if (OldCCValid == Branch.CCValid && OldCCMask == Branch.CCMask)501      continue;502 503    // FIXME: Try combining conditions like X86 does.  Should be easy on Z!504    return false;505  }506 507  return false;508}509 510unsigned SystemZInstrInfo::removeBranch(MachineBasicBlock &MBB,511                                        int *BytesRemoved) const {512  assert(!BytesRemoved && "code size not handled");513 514  // Most of the code and comments here are boilerplate.515  MachineBasicBlock::iterator I = MBB.end();516  unsigned Count = 0;517 518  while (I != MBB.begin()) {519    --I;520    if (I->isDebugInstr())521      continue;522    if (!I->isBranch())523      break;524    if (!getBranchInfo(*I).hasMBBTarget())525      break;526    // Remove the branch.527    I->eraseFromParent();528    I = MBB.end();529    ++Count;530  }531 532  return Count;533}534 535bool SystemZInstrInfo::536reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {537  assert(Cond.size() == 2 && "Invalid condition");538  Cond[1].setImm(Cond[1].getImm() ^ Cond[0].getImm());539  return false;540}541 542unsigned SystemZInstrInfo::insertBranch(MachineBasicBlock &MBB,543                                        MachineBasicBlock *TBB,544                                        MachineBasicBlock *FBB,545                                        ArrayRef<MachineOperand> Cond,546                                        const DebugLoc &DL,547                                        int *BytesAdded) const {548  // In this function we output 32-bit branches, which should always549  // have enough range.  They can be shortened and relaxed by later code550  // in the pipeline, if desired.551 552  // Shouldn't be a fall through.553  assert(TBB && "insertBranch must not be told to insert a fallthrough");554  assert((Cond.size() == 2 || Cond.size() == 0) &&555         "SystemZ branch conditions have one component!");556  assert(!BytesAdded && "code size not handled");557 558  if (Cond.empty()) {559    // Unconditional branch?560    assert(!FBB && "Unconditional branch with multiple successors!");561    BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(TBB);562    return 1;563  }564 565  // Conditional branch.566  unsigned Count = 0;567  unsigned CCValid = Cond[0].getImm();568  unsigned CCMask = Cond[1].getImm();569  BuildMI(&MBB, DL, get(SystemZ::BRC))570    .addImm(CCValid).addImm(CCMask).addMBB(TBB);571  ++Count;572 573  if (FBB) {574    // Two-way Conditional branch. Insert the second branch.575    BuildMI(&MBB, DL, get(SystemZ::J)).addMBB(FBB);576    ++Count;577  }578  return Count;579}580 581bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,582                                      Register &SrcReg2, int64_t &Mask,583                                      int64_t &Value) const {584  assert(MI.isCompare() && "Caller should have checked for a comparison");585 586  if (MI.getNumExplicitOperands() == 2 && MI.getOperand(0).isReg() &&587      MI.getOperand(1).isImm()) {588    SrcReg = MI.getOperand(0).getReg();589    SrcReg2 = 0;590    Value = MI.getOperand(1).getImm();591    Mask = ~0;592    return true;593  }594 595  return false;596}597 598bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,599                                       ArrayRef<MachineOperand> Pred,600                                       Register DstReg, Register TrueReg,601                                       Register FalseReg, int &CondCycles,602                                       int &TrueCycles,603                                       int &FalseCycles) const {604  // Not all subtargets have LOCR instructions.605  if (!STI.hasLoadStoreOnCond())606    return false;607  if (Pred.size() != 2)608    return false;609 610  // Check register classes.611  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();612  const TargetRegisterClass *RC =613    RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));614  if (!RC)615    return false;616 617  // We have LOCR instructions for 32 and 64 bit general purpose registers.618  if ((STI.hasLoadStoreOnCond2() &&619       SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) ||620      SystemZ::GR32BitRegClass.hasSubClassEq(RC) ||621      SystemZ::GR64BitRegClass.hasSubClassEq(RC)) {622    CondCycles = 2;623    TrueCycles = 2;624    FalseCycles = 2;625    return true;626  }627 628  // Can't do anything else.629  return false;630}631 632void SystemZInstrInfo::insertSelect(MachineBasicBlock &MBB,633                                    MachineBasicBlock::iterator I,634                                    const DebugLoc &DL, Register DstReg,635                                    ArrayRef<MachineOperand> Pred,636                                    Register TrueReg,637                                    Register FalseReg) const {638  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();639  const TargetRegisterClass *RC = MRI.getRegClass(DstReg);640 641  assert(Pred.size() == 2 && "Invalid condition");642  unsigned CCValid = Pred[0].getImm();643  unsigned CCMask = Pred[1].getImm();644 645  unsigned Opc;646  if (SystemZ::GRX32BitRegClass.hasSubClassEq(RC)) {647    if (STI.hasMiscellaneousExtensions3())648      Opc = SystemZ::SELRMux;649    else if (STI.hasLoadStoreOnCond2())650      Opc = SystemZ::LOCRMux;651    else {652      Opc = SystemZ::LOCR;653      MRI.constrainRegClass(DstReg, &SystemZ::GR32BitRegClass);654      Register TReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);655      Register FReg = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass);656      BuildMI(MBB, I, DL, get(TargetOpcode::COPY), TReg).addReg(TrueReg);657      BuildMI(MBB, I, DL, get(TargetOpcode::COPY), FReg).addReg(FalseReg);658      TrueReg = TReg;659      FalseReg = FReg;660    }661  } else if (SystemZ::GR64BitRegClass.hasSubClassEq(RC)) {662    if (STI.hasMiscellaneousExtensions3())663      Opc = SystemZ::SELGR;664    else665      Opc = SystemZ::LOCGR;666  } else667    llvm_unreachable("Invalid register class");668 669  BuildMI(MBB, I, DL, get(Opc), DstReg)670    .addReg(FalseReg).addReg(TrueReg)671    .addImm(CCValid).addImm(CCMask);672}673 674bool SystemZInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,675                                     Register Reg,676                                     MachineRegisterInfo *MRI) const {677  unsigned DefOpc = DefMI.getOpcode();678 679  if (DefOpc == SystemZ::VGBM) {680    int64_t ImmVal = DefMI.getOperand(1).getImm();681    if (ImmVal != 0) // TODO: Handle other values682      return false;683 684    // Fold gr128 = COPY (vr128 VGBM imm)685    //686    // %tmp:gr64 = LGHI 0687    // to  gr128 = REG_SEQUENCE %tmp, %tmp688    assert(DefMI.getOperand(0).getReg() == Reg);689 690    if (!UseMI.isCopy())691      return false;692 693    Register CopyDstReg = UseMI.getOperand(0).getReg();694    if (CopyDstReg.isVirtual() &&695        MRI->getRegClass(CopyDstReg) == &SystemZ::GR128BitRegClass &&696        MRI->hasOneNonDBGUse(Reg)) {697      // TODO: Handle physical registers698      // TODO: Handle gr64 uses with subregister indexes699      // TODO: Should this multi-use cases?700      Register TmpReg = MRI->createVirtualRegister(&SystemZ::GR64BitRegClass);701      MachineBasicBlock &MBB = *UseMI.getParent();702 703      loadImmediate(MBB, UseMI.getIterator(), TmpReg, ImmVal);704 705      UseMI.setDesc(get(SystemZ::REG_SEQUENCE));706      UseMI.getOperand(1).setReg(TmpReg);707      MachineInstrBuilder(*MBB.getParent(), &UseMI)708          .addImm(SystemZ::subreg_h64)709          .addReg(TmpReg)710          .addImm(SystemZ::subreg_l64);711 712      if (MRI->use_nodbg_empty(Reg))713        DefMI.eraseFromParent();714      return true;715    }716 717    return false;718  }719 720  if (DefOpc != SystemZ::LHIMux && DefOpc != SystemZ::LHI &&721      DefOpc != SystemZ::LGHI)722    return false;723  if (DefMI.getOperand(0).getReg() != Reg)724    return false;725  int32_t ImmVal = (int32_t)DefMI.getOperand(1).getImm();726 727  unsigned UseOpc = UseMI.getOpcode();728  unsigned NewUseOpc;729  unsigned UseIdx;730  int CommuteIdx = -1;731  bool TieOps = false;732  switch (UseOpc) {733  case SystemZ::SELRMux:734    TieOps = true;735    [[fallthrough]];736  case SystemZ::LOCRMux:737    if (!STI.hasLoadStoreOnCond2())738      return false;739    NewUseOpc = SystemZ::LOCHIMux;740    if (UseMI.getOperand(2).getReg() == Reg)741      UseIdx = 2;742    else if (UseMI.getOperand(1).getReg() == Reg)743      UseIdx = 2, CommuteIdx = 1;744    else745      return false;746    break;747  case SystemZ::SELGR:748    TieOps = true;749    [[fallthrough]];750  case SystemZ::LOCGR:751    if (!STI.hasLoadStoreOnCond2())752      return false;753    NewUseOpc = SystemZ::LOCGHI;754    if (UseMI.getOperand(2).getReg() == Reg)755      UseIdx = 2;756    else if (UseMI.getOperand(1).getReg() == Reg)757      UseIdx = 2, CommuteIdx = 1;758    else759      return false;760    break;761  default:762    return false;763  }764 765  if (CommuteIdx != -1)766    if (!commuteInstruction(UseMI, false, CommuteIdx, UseIdx))767      return false;768 769  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);770  UseMI.setDesc(get(NewUseOpc));771  if (TieOps)772    UseMI.tieOperands(0, 1);773  UseMI.getOperand(UseIdx).ChangeToImmediate(ImmVal);774  if (DeleteDef)775    DefMI.eraseFromParent();776 777  return true;778}779 780bool SystemZInstrInfo::isPredicable(const MachineInstr &MI) const {781  unsigned Opcode = MI.getOpcode();782  if (Opcode == SystemZ::Return ||783      Opcode == SystemZ::Return_XPLINK ||784      Opcode == SystemZ::Trap ||785      Opcode == SystemZ::CallJG ||786      Opcode == SystemZ::CallBR)787    return true;788  return false;789}790 791bool SystemZInstrInfo::792isProfitableToIfCvt(MachineBasicBlock &MBB,793                    unsigned NumCycles, unsigned ExtraPredCycles,794                    BranchProbability Probability) const {795  // Avoid using conditional returns at the end of a loop (since then796  // we'd need to emit an unconditional branch to the beginning anyway,797  // making the loop body longer).  This doesn't apply for low-probability798  // loops (eg. compare-and-swap retry), so just decide based on branch799  // probability instead of looping structure.800  // However, since Compare and Trap instructions cost the same as a regular801  // Compare instruction, we should allow the if conversion to convert this802  // into a Conditional Compare regardless of the branch probability.803  if (MBB.getLastNonDebugInstr()->getOpcode() != SystemZ::Trap &&804      MBB.succ_empty() && Probability < BranchProbability(1, 8))805    return false;806  // For now only convert single instructions.807  return NumCycles == 1;808}809 810bool SystemZInstrInfo::811isProfitableToIfCvt(MachineBasicBlock &TMBB,812                    unsigned NumCyclesT, unsigned ExtraPredCyclesT,813                    MachineBasicBlock &FMBB,814                    unsigned NumCyclesF, unsigned ExtraPredCyclesF,815                    BranchProbability Probability) const {816  // For now avoid converting mutually-exclusive cases.817  return false;818}819 820bool SystemZInstrInfo::821isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles,822                          BranchProbability Probability) const {823  // For now only duplicate single instructions.824  return NumCycles == 1;825}826 827bool SystemZInstrInfo::PredicateInstruction(828    MachineInstr &MI, ArrayRef<MachineOperand> Pred) const {829  assert(Pred.size() == 2 && "Invalid condition");830  unsigned CCValid = Pred[0].getImm();831  unsigned CCMask = Pred[1].getImm();832  assert(CCMask > 0 && CCMask < 15 && "Invalid predicate");833  unsigned Opcode = MI.getOpcode();834  if (Opcode == SystemZ::Trap) {835    MI.setDesc(get(SystemZ::CondTrap));836    MachineInstrBuilder(*MI.getParent()->getParent(), MI)837      .addImm(CCValid).addImm(CCMask)838      .addReg(SystemZ::CC, RegState::Implicit);839    return true;840  }841  if (Opcode == SystemZ::Return || Opcode == SystemZ::Return_XPLINK) {842    MI.setDesc(get(Opcode == SystemZ::Return ? SystemZ::CondReturn843                                             : SystemZ::CondReturn_XPLINK));844    MachineInstrBuilder(*MI.getParent()->getParent(), MI)845        .addImm(CCValid)846        .addImm(CCMask)847        .addReg(SystemZ::CC, RegState::Implicit);848    return true;849  }850  if (Opcode == SystemZ::CallJG) {851    MachineOperand FirstOp = MI.getOperand(0);852    const uint32_t *RegMask = MI.getOperand(1).getRegMask();853    MI.removeOperand(1);854    MI.removeOperand(0);855    MI.setDesc(get(SystemZ::CallBRCL));856    MachineInstrBuilder(*MI.getParent()->getParent(), MI)857        .addImm(CCValid)858        .addImm(CCMask)859        .add(FirstOp)860        .addRegMask(RegMask)861        .addReg(SystemZ::CC, RegState::Implicit);862    return true;863  }864  if (Opcode == SystemZ::CallBR) {865    MachineOperand Target = MI.getOperand(0);866    const uint32_t *RegMask = MI.getOperand(1).getRegMask();867    MI.removeOperand(1);868    MI.removeOperand(0);869    MI.setDesc(get(SystemZ::CallBCR));870    MachineInstrBuilder(*MI.getParent()->getParent(), MI)871      .addImm(CCValid).addImm(CCMask)872      .add(Target)873      .addRegMask(RegMask)874      .addReg(SystemZ::CC, RegState::Implicit);875    return true;876  }877  return false;878}879 880void SystemZInstrInfo::copyPhysReg(MachineBasicBlock &MBB,881                                   MachineBasicBlock::iterator MBBI,882                                   const DebugLoc &DL, Register DestReg,883                                   Register SrcReg, bool KillSrc,884                                   bool RenamableDest,885                                   bool RenamableSrc) const {886  // Split 128-bit GPR moves into two 64-bit moves. Add implicit uses of the887  // super register in case one of the subregs is undefined.888  // This handles ADDR128 too.889  if (SystemZ::GR128BitRegClass.contains(DestReg, SrcReg)) {890    copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_h64),891                RI.getSubReg(SrcReg, SystemZ::subreg_h64), KillSrc);892    MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI))893      .addReg(SrcReg, RegState::Implicit);894    copyPhysReg(MBB, MBBI, DL, RI.getSubReg(DestReg, SystemZ::subreg_l64),895                RI.getSubReg(SrcReg, SystemZ::subreg_l64), KillSrc);896    MachineInstrBuilder(*MBB.getParent(), std::prev(MBBI))897      .addReg(SrcReg, (getKillRegState(KillSrc) | RegState::Implicit));898    return;899  }900 901  if (SystemZ::GRX32BitRegClass.contains(DestReg, SrcReg)) {902    emitGRX32Move(MBB, MBBI, DL, DestReg, SrcReg, SystemZ::LR, 32, KillSrc,903                  false);904    return;905  }906 907  // Move 128-bit floating-point values between VR128 and FP128.908  if (SystemZ::VR128BitRegClass.contains(DestReg) &&909      SystemZ::FP128BitRegClass.contains(SrcReg)) {910    MCRegister SrcRegHi =911        RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_h64),912                               SystemZ::subreg_h64, &SystemZ::VR128BitRegClass);913    MCRegister SrcRegLo =914        RI.getMatchingSuperReg(RI.getSubReg(SrcReg, SystemZ::subreg_l64),915                               SystemZ::subreg_h64, &SystemZ::VR128BitRegClass);916 917    BuildMI(MBB, MBBI, DL, get(SystemZ::VMRHG), DestReg)918      .addReg(SrcRegHi, getKillRegState(KillSrc))919      .addReg(SrcRegLo, getKillRegState(KillSrc));920    return;921  }922  if (SystemZ::FP128BitRegClass.contains(DestReg) &&923      SystemZ::VR128BitRegClass.contains(SrcReg)) {924    MCRegister DestRegHi =925        RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_h64),926                               SystemZ::subreg_h64, &SystemZ::VR128BitRegClass);927    MCRegister DestRegLo =928        RI.getMatchingSuperReg(RI.getSubReg(DestReg, SystemZ::subreg_l64),929                               SystemZ::subreg_h64, &SystemZ::VR128BitRegClass);930 931    if (DestRegHi != SrcReg.asMCReg())932      copyPhysReg(MBB, MBBI, DL, DestRegHi, SrcReg, false);933    BuildMI(MBB, MBBI, DL, get(SystemZ::VREPG), DestRegLo)934      .addReg(SrcReg, getKillRegState(KillSrc)).addImm(1);935    return;936  }937 938  if (SystemZ::FP128BitRegClass.contains(DestReg) &&939      SystemZ::GR128BitRegClass.contains(SrcReg)) {940    MCRegister DestRegHi = RI.getSubReg(DestReg, SystemZ::subreg_h64);941    MCRegister DestRegLo = RI.getSubReg(DestReg, SystemZ::subreg_l64);942    MCRegister SrcRegHi = RI.getSubReg(SrcReg, SystemZ::subreg_h64);943    MCRegister SrcRegLo = RI.getSubReg(SrcReg, SystemZ::subreg_l64);944 945    BuildMI(MBB, MBBI, DL, get(SystemZ::LDGR), DestRegHi)946        .addReg(SrcRegHi)947        .addReg(DestReg, RegState::ImplicitDefine);948 949    BuildMI(MBB, MBBI, DL, get(SystemZ::LDGR), DestRegLo)950        .addReg(SrcRegLo, getKillRegState(KillSrc));951    return;952  }953 954  // Move CC value from a GR32.955  if (DestReg == SystemZ::CC) {956    unsigned Opcode =957      SystemZ::GR32BitRegClass.contains(SrcReg) ? SystemZ::TMLH : SystemZ::TMHH;958    BuildMI(MBB, MBBI, DL, get(Opcode))959      .addReg(SrcReg, getKillRegState(KillSrc))960      .addImm(3 << (SystemZ::IPM_CC - 16));961    return;962  }963 964  if (SystemZ::GR128BitRegClass.contains(DestReg) &&965      SystemZ::VR128BitRegClass.contains(SrcReg)) {966    MCRegister DestH64 = RI.getSubReg(DestReg, SystemZ::subreg_h64);967    MCRegister DestL64 = RI.getSubReg(DestReg, SystemZ::subreg_l64);968 969    BuildMI(MBB, MBBI, DL, get(SystemZ::VLGVG), DestH64)970        .addReg(SrcReg)971        .addReg(SystemZ::NoRegister)972        .addImm(0)973        .addDef(DestReg, RegState::Implicit);974    BuildMI(MBB, MBBI, DL, get(SystemZ::VLGVG), DestL64)975        .addReg(SrcReg, getKillRegState(KillSrc))976        .addReg(SystemZ::NoRegister)977        .addImm(1);978    return;979  }980 981  if (SystemZ::VR128BitRegClass.contains(DestReg) &&982      SystemZ::GR128BitRegClass.contains(SrcReg)) {983    BuildMI(MBB, MBBI, DL, get(SystemZ::VLVGP), DestReg)984        .addReg(RI.getSubReg(SrcReg, SystemZ::subreg_h64))985        .addReg(RI.getSubReg(SrcReg, SystemZ::subreg_l64));986    return;987  }988 989  // Everything else needs only one instruction.990  unsigned Opcode;991  if (SystemZ::GR64BitRegClass.contains(DestReg, SrcReg))992    Opcode = SystemZ::LGR;993  else if (SystemZ::FP16BitRegClass.contains(DestReg, SrcReg))994    Opcode = STI.hasVector() ? SystemZ::LDR16 : SystemZ::LER16;995  else if (SystemZ::FP32BitRegClass.contains(DestReg, SrcReg))996    // For z13 we prefer LDR over LER to avoid partial register dependencies.997    Opcode = STI.hasVector() ? SystemZ::LDR32 : SystemZ::LER;998  else if (SystemZ::FP64BitRegClass.contains(DestReg, SrcReg))999    Opcode = SystemZ::LDR;1000  else if (SystemZ::FP128BitRegClass.contains(DestReg, SrcReg))1001    Opcode = SystemZ::LXR;1002  else if (SystemZ::VR32BitRegClass.contains(DestReg, SrcReg))1003    Opcode = SystemZ::VLR32;1004  else if (SystemZ::VR64BitRegClass.contains(DestReg, SrcReg))1005    Opcode = SystemZ::VLR64;1006  else if (SystemZ::VR128BitRegClass.contains(DestReg, SrcReg))1007    Opcode = SystemZ::VLR;1008  else if (SystemZ::AR32BitRegClass.contains(DestReg, SrcReg))1009    Opcode = SystemZ::CPYA;1010  else if (SystemZ::GR64BitRegClass.contains(DestReg) &&1011           SystemZ::FP64BitRegClass.contains(SrcReg))1012    Opcode = SystemZ::LGDR;1013  else if (SystemZ::FP64BitRegClass.contains(DestReg) &&1014           SystemZ::GR64BitRegClass.contains(SrcReg))1015    Opcode = SystemZ::LDGR;1016  else1017    llvm_unreachable("Impossible reg-to-reg copy");1018 1019  BuildMI(MBB, MBBI, DL, get(Opcode), DestReg)1020    .addReg(SrcReg, getKillRegState(KillSrc));1021}1022 1023void SystemZInstrInfo::storeRegToStackSlot(1024    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,1025    bool isKill, int FrameIdx, const TargetRegisterClass *RC,1026 1027    Register VReg, MachineInstr::MIFlag Flags) const {1028  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();1029 1030  // Callers may expect a single instruction, so keep 128-bit moves1031  // together for now and lower them after register allocation.1032  unsigned LoadOpcode, StoreOpcode;1033  getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode);1034  addFrameReference(BuildMI(MBB, MBBI, DL, get(StoreOpcode))1035                        .addReg(SrcReg, getKillRegState(isKill)),1036                    FrameIdx);1037}1038 1039void SystemZInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,1040                                            MachineBasicBlock::iterator MBBI,1041                                            Register DestReg, int FrameIdx,1042                                            const TargetRegisterClass *RC,1043                                            Register VReg,1044                                            MachineInstr::MIFlag Flags) const {1045  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();1046 1047  // Callers may expect a single instruction, so keep 128-bit moves1048  // together for now and lower them after register allocation.1049  unsigned LoadOpcode, StoreOpcode;1050  getLoadStoreOpcodes(RC, LoadOpcode, StoreOpcode);1051  addFrameReference(BuildMI(MBB, MBBI, DL, get(LoadOpcode), DestReg),1052                    FrameIdx);1053}1054 1055// Return true if MI is a simple load or store with a 12-bit displacement1056// and no index.  Flag is SimpleBDXLoad for loads and SimpleBDXStore for stores.1057static bool isSimpleBD12Move(const MachineInstr *MI, unsigned Flag) {1058  const MCInstrDesc &MCID = MI->getDesc();1059  return ((MCID.TSFlags & Flag) &&1060          isUInt<12>(MI->getOperand(2).getImm()) &&1061          MI->getOperand(3).getReg() == 0);1062}1063 1064namespace {1065 1066struct LogicOp {1067  LogicOp() = default;1068  LogicOp(unsigned regSize, unsigned immLSB, unsigned immSize)1069    : RegSize(regSize), ImmLSB(immLSB), ImmSize(immSize) {}1070 1071  explicit operator bool() const { return RegSize; }1072 1073  unsigned RegSize = 0;1074  unsigned ImmLSB = 0;1075  unsigned ImmSize = 0;1076};1077 1078} // end anonymous namespace1079 1080static LogicOp interpretAndImmediate(unsigned Opcode) {1081  switch (Opcode) {1082  case SystemZ::NILMux: return LogicOp(32,  0, 16);1083  case SystemZ::NIHMux: return LogicOp(32, 16, 16);1084  case SystemZ::NILL64: return LogicOp(64,  0, 16);1085  case SystemZ::NILH64: return LogicOp(64, 16, 16);1086  case SystemZ::NIHL64: return LogicOp(64, 32, 16);1087  case SystemZ::NIHH64: return LogicOp(64, 48, 16);1088  case SystemZ::NIFMux: return LogicOp(32,  0, 32);1089  case SystemZ::NILF64: return LogicOp(64,  0, 32);1090  case SystemZ::NIHF64: return LogicOp(64, 32, 32);1091  default:              return LogicOp();1092  }1093}1094 1095static void transferDeadCC(MachineInstr *OldMI, MachineInstr *NewMI) {1096  if (OldMI->registerDefIsDead(SystemZ::CC, /*TRI=*/nullptr)) {1097    MachineOperand *CCDef =1098        NewMI->findRegisterDefOperand(SystemZ::CC, /*TRI=*/nullptr);1099    if (CCDef != nullptr)1100      CCDef->setIsDead(true);1101  }1102}1103 1104static void transferMIFlag(MachineInstr *OldMI, MachineInstr *NewMI,1105                           MachineInstr::MIFlag Flag) {1106  if (OldMI->getFlag(Flag))1107    NewMI->setFlag(Flag);1108}1109 1110MachineInstr *1111SystemZInstrInfo::convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,1112                                        LiveIntervals *LIS) const {1113  MachineBasicBlock *MBB = MI.getParent();1114 1115  // Try to convert an AND into an RISBG-type instruction.1116  // TODO: It might be beneficial to select RISBG and shorten to AND instead.1117  if (LogicOp And = interpretAndImmediate(MI.getOpcode())) {1118    uint64_t Imm = MI.getOperand(2).getImm() << And.ImmLSB;1119    // AND IMMEDIATE leaves the other bits of the register unchanged.1120    Imm |= allOnes(And.RegSize) & ~(allOnes(And.ImmSize) << And.ImmLSB);1121    unsigned Start, End;1122    if (isRxSBGMask(Imm, And.RegSize, Start, End)) {1123      unsigned NewOpcode;1124      if (And.RegSize == 64) {1125        NewOpcode = SystemZ::RISBG;1126        // Prefer RISBGN if available, since it does not clobber CC.1127        if (STI.hasMiscellaneousExtensions())1128          NewOpcode = SystemZ::RISBGN;1129      } else {1130        NewOpcode = SystemZ::RISBMux;1131        Start &= 31;1132        End &= 31;1133      }1134      MachineOperand &Dest = MI.getOperand(0);1135      MachineOperand &Src = MI.getOperand(1);1136      MachineInstrBuilder MIB =1137          BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpcode))1138              .add(Dest)1139              .addReg(0)1140              .addReg(Src.getReg(), getKillRegState(Src.isKill()),1141                      Src.getSubReg())1142              .addImm(Start)1143              .addImm(End + 128)1144              .addImm(0);1145      if (LV) {1146        unsigned NumOps = MI.getNumOperands();1147        for (unsigned I = 1; I < NumOps; ++I) {1148          MachineOperand &Op = MI.getOperand(I);1149          if (Op.isReg() && Op.isKill())1150            LV->replaceKillInstruction(Op.getReg(), MI, *MIB);1151        }1152      }1153      if (LIS)1154        LIS->ReplaceMachineInstrInMaps(MI, *MIB);1155      transferDeadCC(&MI, MIB);1156      return MIB;1157    }1158  }1159  return nullptr;1160}1161 1162bool SystemZInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,1163                                                   bool Invert) const {1164  unsigned Opc = Inst.getOpcode();1165  if (Invert) {1166    auto InverseOpcode = getInverseOpcode(Opc);1167    if (!InverseOpcode)1168      return false;1169    Opc = *InverseOpcode;1170  }1171 1172  switch (Opc) {1173  default:1174    break;1175  // Adds and multiplications.1176  case SystemZ::WFADB:1177  case SystemZ::WFASB:1178  case SystemZ::WFAXB:1179  case SystemZ::VFADB:1180  case SystemZ::VFASB:1181  case SystemZ::WFMDB:1182  case SystemZ::WFMSB:1183  case SystemZ::WFMXB:1184  case SystemZ::VFMDB:1185  case SystemZ::VFMSB:1186    return (Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&1187            Inst.getFlag(MachineInstr::MIFlag::FmNsz));1188  }1189 1190  return false;1191}1192 1193std::optional<unsigned>1194SystemZInstrInfo::getInverseOpcode(unsigned Opcode) const {1195  // fadd => fsub1196  switch (Opcode) {1197  case SystemZ::WFADB:1198    return SystemZ::WFSDB;1199  case SystemZ::WFASB:1200    return SystemZ::WFSSB;1201  case SystemZ::WFAXB:1202    return SystemZ::WFSXB;1203  case SystemZ::VFADB:1204    return SystemZ::VFSDB;1205  case SystemZ::VFASB:1206    return SystemZ::VFSSB;1207  // fsub => fadd1208  case SystemZ::WFSDB:1209    return SystemZ::WFADB;1210  case SystemZ::WFSSB:1211    return SystemZ::WFASB;1212  case SystemZ::WFSXB:1213    return SystemZ::WFAXB;1214  case SystemZ::VFSDB:1215    return SystemZ::VFADB;1216  case SystemZ::VFSSB:1217    return SystemZ::VFASB;1218  default:1219    return std::nullopt;1220  }1221}1222 1223MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(1224    MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,1225    MachineBasicBlock::iterator InsertPt, int FrameIndex,1226    LiveIntervals *LIS, VirtRegMap *VRM) const {1227  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();1228  MachineRegisterInfo &MRI = MF.getRegInfo();1229  const MachineFrameInfo &MFI = MF.getFrameInfo();1230  unsigned Size = MFI.getObjectSize(FrameIndex);1231  unsigned Opcode = MI.getOpcode();1232 1233  // Check CC liveness if new instruction introduces a dead def of CC.1234  SlotIndex MISlot = SlotIndex();1235  LiveRange *CCLiveRange = nullptr;1236  bool CCLiveAtMI = true;1237  if (LIS) {1238    MISlot = LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot();1239    auto CCUnits = TRI->regunits(MCRegister::from(SystemZ::CC));1240    assert(range_size(CCUnits) == 1 && "CC only has one reg unit.");1241    CCLiveRange = &LIS->getRegUnit(*CCUnits.begin());1242    CCLiveAtMI = CCLiveRange->liveAt(MISlot);1243  }1244 1245  if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {1246    if (!CCLiveAtMI && (Opcode == SystemZ::LA || Opcode == SystemZ::LAY) &&1247        isInt<8>(MI.getOperand(2).getImm()) && !MI.getOperand(3).getReg()) {1248      // LA(Y) %reg, CONST(%reg) -> AGSI %mem, CONST1249      MachineInstr *BuiltMI = BuildMI(*InsertPt->getParent(), InsertPt,1250                                      MI.getDebugLoc(), get(SystemZ::AGSI))1251        .addFrameIndex(FrameIndex)1252        .addImm(0)1253        .addImm(MI.getOperand(2).getImm());1254      BuiltMI->findRegisterDefOperand(SystemZ::CC, /*TRI=*/nullptr)1255          ->setIsDead(true);1256      CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator());1257      return BuiltMI;1258    }1259    return nullptr;1260  }1261 1262  // All other cases require a single operand.1263  if (Ops.size() != 1)1264    return nullptr;1265 1266  unsigned OpNum = Ops[0];1267  const TargetRegisterClass *RC =1268      MF.getRegInfo().getRegClass(MI.getOperand(OpNum).getReg());1269  assert((Size * 8 == TRI->getRegSizeInBits(*RC) ||1270          (RC == &SystemZ::FP16BitRegClass && Size == 4 && !STI.hasVector())) &&1271         "Invalid size combination");1272  (void)RC;1273 1274  if ((Opcode == SystemZ::AHI || Opcode == SystemZ::AGHI) && OpNum == 0 &&1275      isInt<8>(MI.getOperand(2).getImm())) {1276    // A(G)HI %reg, CONST -> A(G)SI %mem, CONST1277    Opcode = (Opcode == SystemZ::AHI ? SystemZ::ASI : SystemZ::AGSI);1278    MachineInstr *BuiltMI =1279        BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode))1280            .addFrameIndex(FrameIndex)1281            .addImm(0)1282            .addImm(MI.getOperand(2).getImm());1283    transferDeadCC(&MI, BuiltMI);1284    transferMIFlag(&MI, BuiltMI, MachineInstr::NoSWrap);1285    return BuiltMI;1286  }1287 1288  if ((Opcode == SystemZ::ALFI && OpNum == 0 &&1289       isInt<8>((int32_t)MI.getOperand(2).getImm())) ||1290      (Opcode == SystemZ::ALGFI && OpNum == 0 &&1291       isInt<8>(MI.getOperand(2).getImm()))) {1292    // AL(G)FI %reg, CONST -> AL(G)SI %mem, CONST1293    Opcode = (Opcode == SystemZ::ALFI ? SystemZ::ALSI : SystemZ::ALGSI);1294    MachineInstr *BuiltMI =1295        BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode))1296            .addFrameIndex(FrameIndex)1297            .addImm(0)1298            .addImm((int8_t)MI.getOperand(2).getImm());1299    transferDeadCC(&MI, BuiltMI);1300    return BuiltMI;1301  }1302 1303  if ((Opcode == SystemZ::SLFI && OpNum == 0 &&1304       isInt<8>((int32_t)-MI.getOperand(2).getImm())) ||1305      (Opcode == SystemZ::SLGFI && OpNum == 0 &&1306       isInt<8>((-MI.getOperand(2).getImm())))) {1307    // SL(G)FI %reg, CONST -> AL(G)SI %mem, -CONST1308    Opcode = (Opcode == SystemZ::SLFI ? SystemZ::ALSI : SystemZ::ALGSI);1309    MachineInstr *BuiltMI =1310        BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(), get(Opcode))1311            .addFrameIndex(FrameIndex)1312            .addImm(0)1313            .addImm((int8_t)-MI.getOperand(2).getImm());1314    transferDeadCC(&MI, BuiltMI);1315    return BuiltMI;1316  }1317 1318  unsigned MemImmOpc = 0;1319  switch (Opcode) {1320  case SystemZ::LHIMux:1321  case SystemZ::LHI:    MemImmOpc = SystemZ::MVHI;  break;1322  case SystemZ::LGHI:   MemImmOpc = SystemZ::MVGHI; break;1323  case SystemZ::CHIMux:1324  case SystemZ::CHI:    MemImmOpc = SystemZ::CHSI;  break;1325  case SystemZ::CGHI:   MemImmOpc = SystemZ::CGHSI; break;1326  case SystemZ::CLFIMux:1327  case SystemZ::CLFI:1328    if (isUInt<16>(MI.getOperand(1).getImm()))1329      MemImmOpc = SystemZ::CLFHSI;1330    break;1331  case SystemZ::CLGFI:1332    if (isUInt<16>(MI.getOperand(1).getImm()))1333      MemImmOpc = SystemZ::CLGHSI;1334    break;1335  default: break;1336  }1337  if (MemImmOpc)1338    return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),1339                   get(MemImmOpc))1340               .addFrameIndex(FrameIndex)1341               .addImm(0)1342               .addImm(MI.getOperand(1).getImm());1343 1344  if (Opcode == SystemZ::LGDR || Opcode == SystemZ::LDGR) {1345    bool Op0IsGPR = (Opcode == SystemZ::LGDR);1346    bool Op1IsGPR = (Opcode == SystemZ::LDGR);1347    // If we're spilling the destination of an LDGR or LGDR, store the1348    // source register instead.1349    if (OpNum == 0) {1350      unsigned StoreOpcode = Op1IsGPR ? SystemZ::STG : SystemZ::STD;1351      return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),1352                     get(StoreOpcode))1353          .add(MI.getOperand(1))1354          .addFrameIndex(FrameIndex)1355          .addImm(0)1356          .addReg(0);1357    }1358    // If we're spilling the source of an LDGR or LGDR, load the1359    // destination register instead.1360    if (OpNum == 1) {1361      unsigned LoadOpcode = Op0IsGPR ? SystemZ::LG : SystemZ::LD;1362      return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),1363                     get(LoadOpcode))1364        .add(MI.getOperand(0))1365        .addFrameIndex(FrameIndex)1366        .addImm(0)1367        .addReg(0);1368    }1369  }1370 1371  // Look for cases where the source of a simple store or the destination1372  // of a simple load is being spilled.  Try to use MVC instead.1373  //1374  // Although MVC is in practice a fast choice in these cases, it is still1375  // logically a bytewise copy.  This means that we cannot use it if the1376  // load or store is volatile.  We also wouldn't be able to use MVC if1377  // the two memories partially overlap, but that case cannot occur here,1378  // because we know that one of the memories is a full frame index.1379  //1380  // For performance reasons, we also want to avoid using MVC if the addresses1381  // might be equal.  We don't worry about that case here, because spill slot1382  // coloring happens later, and because we have special code to remove1383  // MVCs that turn out to be redundant.1384  if (OpNum == 0 && MI.hasOneMemOperand()) {1385    MachineMemOperand *MMO = *MI.memoperands_begin();1386    if (MMO->getSize() == Size && !MMO->isVolatile() && !MMO->isAtomic()) {1387      // Handle conversion of loads.1388      if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXLoad)) {1389        return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),1390                       get(SystemZ::MVC))1391            .addFrameIndex(FrameIndex)1392            .addImm(0)1393            .addImm(Size)1394            .add(MI.getOperand(1))1395            .addImm(MI.getOperand(2).getImm())1396            .addMemOperand(MMO);1397      }1398      // Handle conversion of stores.1399      if (isSimpleBD12Move(&MI, SystemZII::SimpleBDXStore)) {1400        return BuildMI(*InsertPt->getParent(), InsertPt, MI.getDebugLoc(),1401                       get(SystemZ::MVC))1402            .add(MI.getOperand(1))1403            .addImm(MI.getOperand(2).getImm())1404            .addImm(Size)1405            .addFrameIndex(FrameIndex)1406            .addImm(0)1407            .addMemOperand(MMO);1408      }1409    }1410  }1411 1412  // If the spilled operand is the final one or the instruction is1413  // commutable, try to change <INSN>R into <INSN>.  Don't introduce a def of1414  // CC if it is live and MI does not define it.1415  unsigned NumOps = MI.getNumExplicitOperands();1416  int MemOpcode = SystemZ::getMemOpcode(Opcode);1417  if (MemOpcode == -1 ||1418      (CCLiveAtMI && !MI.definesRegister(SystemZ::CC, /*TRI=*/nullptr) &&1419       get(MemOpcode).hasImplicitDefOfPhysReg(SystemZ::CC)))1420    return nullptr;1421 1422  // Check if all other vregs have a usable allocation in the case of vector1423  // to FP conversion.1424  const MCInstrDesc &MCID = MI.getDesc();1425  for (unsigned I = 0, E = MCID.getNumOperands(); I != E; ++I) {1426    const MCOperandInfo &MCOI = MCID.operands()[I];1427    if (MCOI.OperandType != MCOI::OPERAND_REGISTER || I == OpNum)1428      continue;1429    const TargetRegisterClass *RC = TRI->getRegClass(MCOI.RegClass);1430    if (RC == &SystemZ::VR32BitRegClass || RC == &SystemZ::VR64BitRegClass) {1431      Register Reg = MI.getOperand(I).getReg();1432      Register PhysReg = Reg.isVirtual()1433                             ? (VRM ? Register(VRM->getPhys(Reg)) : Register())1434                             : Reg;1435      if (!PhysReg ||1436          !(SystemZ::FP32BitRegClass.contains(PhysReg) ||1437            SystemZ::FP64BitRegClass.contains(PhysReg) ||1438            SystemZ::VF128BitRegClass.contains(PhysReg)))1439        return nullptr;1440    }1441  }1442  // Fused multiply and add/sub need to have the same dst and accumulator reg.1443  bool FusedFPOp = (Opcode == SystemZ::WFMADB || Opcode == SystemZ::WFMASB ||1444                    Opcode == SystemZ::WFMSDB || Opcode == SystemZ::WFMSSB);1445  if (FusedFPOp) {1446    Register DstReg = VRM->getPhys(MI.getOperand(0).getReg());1447    Register AccReg = VRM->getPhys(MI.getOperand(3).getReg());1448    if (OpNum == 0 || OpNum == 3 || DstReg != AccReg)1449      return nullptr;1450  }1451 1452  // Try to swap compare operands if possible.1453  bool NeedsCommute = false;1454  if ((MI.getOpcode() == SystemZ::CR || MI.getOpcode() == SystemZ::CGR ||1455       MI.getOpcode() == SystemZ::CLR || MI.getOpcode() == SystemZ::CLGR ||1456       MI.getOpcode() == SystemZ::WFCDB || MI.getOpcode() == SystemZ::WFCSB ||1457       MI.getOpcode() == SystemZ::WFKDB || MI.getOpcode() == SystemZ::WFKSB) &&1458      OpNum == 0 && prepareCompareSwapOperands(MI))1459    NeedsCommute = true;1460 1461  bool CCOperands = false;1462  if (MI.getOpcode() == SystemZ::LOCRMux || MI.getOpcode() == SystemZ::LOCGR ||1463      MI.getOpcode() == SystemZ::SELRMux || MI.getOpcode() == SystemZ::SELGR) {1464    assert(MI.getNumOperands() == 6 && NumOps == 5 &&1465           "LOCR/SELR instruction operands corrupt?");1466    NumOps -= 2;1467    CCOperands = true;1468  }1469 1470  // See if this is a 3-address instruction that is convertible to 2-address1471  // and suitable for folding below.  Only try this with virtual registers1472  // and a provided VRM (during regalloc).1473  if (NumOps == 3 && SystemZ::getTargetMemOpcode(MemOpcode) != -1) {1474    if (VRM == nullptr)1475      return nullptr;1476    else {1477      Register DstReg = MI.getOperand(0).getReg();1478      Register DstPhys =1479          (DstReg.isVirtual() ? Register(VRM->getPhys(DstReg)) : DstReg);1480      Register SrcReg = (OpNum == 2 ? MI.getOperand(1).getReg()1481                                    : ((OpNum == 1 && MI.isCommutable())1482                                           ? MI.getOperand(2).getReg()1483                                           : Register()));1484      if (DstPhys && !SystemZ::GRH32BitRegClass.contains(DstPhys) && SrcReg &&1485          SrcReg.isVirtual() && DstPhys == VRM->getPhys(SrcReg))1486        NeedsCommute = (OpNum == 1);1487      else1488        return nullptr;1489    }1490  }1491 1492  if ((OpNum == NumOps - 1) || NeedsCommute || FusedFPOp) {1493    const MCInstrDesc &MemDesc = get(MemOpcode);1494    uint64_t AccessBytes = SystemZII::getAccessSize(MemDesc.TSFlags);1495    assert(AccessBytes != 0 && "Size of access should be known");1496    assert(AccessBytes <= Size && "Access outside the frame index");1497    uint64_t Offset = Size - AccessBytes;1498    MachineInstrBuilder MIB = BuildMI(*InsertPt->getParent(), InsertPt,1499                                      MI.getDebugLoc(), get(MemOpcode));1500    if (MI.isCompare()) {1501      assert(NumOps == 2 && "Expected 2 register operands for a compare.");1502      MIB.add(MI.getOperand(NeedsCommute ? 1 : 0));1503    }1504    else if (FusedFPOp) {1505      MIB.add(MI.getOperand(0));1506      MIB.add(MI.getOperand(3));1507      MIB.add(MI.getOperand(OpNum == 1 ? 2 : 1));1508    }1509    else {1510      MIB.add(MI.getOperand(0));1511      if (NeedsCommute)1512        MIB.add(MI.getOperand(2));1513      else1514        for (unsigned I = 1; I < OpNum; ++I)1515          MIB.add(MI.getOperand(I));1516    }1517    MIB.addFrameIndex(FrameIndex).addImm(Offset);1518    if (MemDesc.TSFlags & SystemZII::HasIndex)1519      MIB.addReg(0);1520    if (CCOperands) {1521      unsigned CCValid = MI.getOperand(NumOps).getImm();1522      unsigned CCMask = MI.getOperand(NumOps + 1).getImm();1523      MIB.addImm(CCValid);1524      MIB.addImm(NeedsCommute ? CCMask ^ CCValid : CCMask);1525    }1526    if (MIB->definesRegister(SystemZ::CC, /*TRI=*/nullptr) &&1527        (!MI.definesRegister(SystemZ::CC, /*TRI=*/nullptr) ||1528         MI.registerDefIsDead(SystemZ::CC, /*TRI=*/nullptr))) {1529      MIB->addRegisterDead(SystemZ::CC, TRI);1530      if (CCLiveRange)1531        CCLiveRange->createDeadDef(MISlot, LIS->getVNInfoAllocator());1532    }1533    // Constrain the register classes if converted from a vector opcode. The1534    // allocated regs are in an FP reg-class per previous check above.1535    for (const MachineOperand &MO : MIB->operands())1536      if (MO.isReg() && MO.getReg().isVirtual()) {1537        Register Reg = MO.getReg();1538        if (MRI.getRegClass(Reg) == &SystemZ::VR32BitRegClass)1539          MRI.setRegClass(Reg, &SystemZ::FP32BitRegClass);1540        else if (MRI.getRegClass(Reg) == &SystemZ::VR64BitRegClass)1541          MRI.setRegClass(Reg, &SystemZ::FP64BitRegClass);1542        else if (MRI.getRegClass(Reg) == &SystemZ::VR128BitRegClass)1543          MRI.setRegClass(Reg, &SystemZ::VF128BitRegClass);1544      }1545 1546    transferDeadCC(&MI, MIB);1547    transferMIFlag(&MI, MIB, MachineInstr::NoSWrap);1548    transferMIFlag(&MI, MIB, MachineInstr::NoFPExcept);1549    return MIB;1550  }1551 1552  return nullptr;1553}1554 1555MachineInstr *SystemZInstrInfo::foldMemoryOperandImpl(1556    MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,1557    MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,1558    LiveIntervals *LIS) const {1559  MachineRegisterInfo *MRI = &MF.getRegInfo();1560  MachineBasicBlock *MBB = MI.getParent();1561 1562  // For reassociable FP operations, any loads have been purposefully left1563  // unfolded so that MachineCombiner can do its work on reg/reg1564  // opcodes. After that, as many loads as possible are now folded.1565  // TODO: This may be beneficial with other opcodes as well as machine-sink1566  // can move loads close to their user in a different MBB, which the isel1567  // matcher did not see.1568  unsigned LoadOpc = 0;1569  unsigned RegMemOpcode = 0;1570  const TargetRegisterClass *FPRC = nullptr;1571  RegMemOpcode = MI.getOpcode() == SystemZ::WFADB   ? SystemZ::ADB1572                 : MI.getOpcode() == SystemZ::WFSDB ? SystemZ::SDB1573                 : MI.getOpcode() == SystemZ::WFMDB ? SystemZ::MDB1574                                                    : 0;1575  if (RegMemOpcode) {1576    LoadOpc = SystemZ::VL64;1577    FPRC = &SystemZ::FP64BitRegClass;1578  } else {1579    RegMemOpcode = MI.getOpcode() == SystemZ::WFASB   ? SystemZ::AEB1580                   : MI.getOpcode() == SystemZ::WFSSB ? SystemZ::SEB1581                   : MI.getOpcode() == SystemZ::WFMSB ? SystemZ::MEEB1582                                                      : 0;1583    if (RegMemOpcode) {1584      LoadOpc = SystemZ::VL32;1585      FPRC = &SystemZ::FP32BitRegClass;1586    }1587  }1588  if (!RegMemOpcode || LoadMI.getOpcode() != LoadOpc)1589    return nullptr;1590 1591  // If RegMemOpcode clobbers CC, first make sure CC is not live at this point.1592  if (get(RegMemOpcode).hasImplicitDefOfPhysReg(SystemZ::CC)) {1593    assert(LoadMI.getParent() == MI.getParent() && "Assuming a local fold.");1594    assert(LoadMI != InsertPt && "Assuming InsertPt not to be first in MBB.");1595    for (MachineBasicBlock::iterator MII = std::prev(InsertPt);;1596         --MII) {1597      if (MII->definesRegister(SystemZ::CC, /*TRI=*/nullptr)) {1598        if (!MII->registerDefIsDead(SystemZ::CC, /*TRI=*/nullptr))1599          return nullptr;1600        break;1601      }1602      if (MII == MBB->begin()) {1603        if (MBB->isLiveIn(SystemZ::CC))1604          return nullptr;1605        break;1606      }1607    }1608  }1609 1610  Register FoldAsLoadDefReg = LoadMI.getOperand(0).getReg();1611  if (Ops.size() != 1 || FoldAsLoadDefReg != MI.getOperand(Ops[0]).getReg())1612    return nullptr;1613  Register DstReg = MI.getOperand(0).getReg();1614  MachineOperand LHS = MI.getOperand(1);1615  MachineOperand RHS = MI.getOperand(2);1616  MachineOperand &RegMO = RHS.getReg() == FoldAsLoadDefReg ? LHS : RHS;1617  if ((RegMemOpcode == SystemZ::SDB || RegMemOpcode == SystemZ::SEB) &&1618      FoldAsLoadDefReg != RHS.getReg())1619    return nullptr;1620 1621  MachineOperand &Base = LoadMI.getOperand(1);1622  MachineOperand &Disp = LoadMI.getOperand(2);1623  MachineOperand &Indx = LoadMI.getOperand(3);1624  MachineInstrBuilder MIB =1625      BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(RegMemOpcode), DstReg)1626          .add(RegMO)1627          .add(Base)1628          .add(Disp)1629          .add(Indx);1630  MIB->addRegisterDead(SystemZ::CC, &RI);1631  MRI->setRegClass(DstReg, FPRC);1632  MRI->setRegClass(RegMO.getReg(), FPRC);1633  transferMIFlag(&MI, MIB, MachineInstr::NoFPExcept);1634 1635  return MIB;1636}1637 1638bool SystemZInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {1639  switch (MI.getOpcode()) {1640  case SystemZ::L128:1641    splitMove(MI, SystemZ::LG);1642    return true;1643 1644  case SystemZ::ST128:1645    splitMove(MI, SystemZ::STG);1646    return true;1647 1648  case SystemZ::LX:1649    splitMove(MI, SystemZ::LD);1650    return true;1651 1652  case SystemZ::STX:1653    splitMove(MI, SystemZ::STD);1654    return true;1655 1656  case SystemZ::LBMux:1657    expandRXYPseudo(MI, SystemZ::LB, SystemZ::LBH);1658    return true;1659 1660  case SystemZ::LHMux:1661    expandRXYPseudo(MI, SystemZ::LH, SystemZ::LHH);1662    return true;1663 1664  case SystemZ::LLCRMux:1665    expandZExtPseudo(MI, SystemZ::LLCR, 8);1666    return true;1667 1668  case SystemZ::LLHRMux:1669    expandZExtPseudo(MI, SystemZ::LLHR, 16);1670    return true;1671 1672  case SystemZ::LLCMux:1673    expandRXYPseudo(MI, SystemZ::LLC, SystemZ::LLCH);1674    return true;1675 1676  case SystemZ::LLHMux:1677    expandRXYPseudo(MI, SystemZ::LLH, SystemZ::LLHH);1678    return true;1679 1680  case SystemZ::LMux:1681    expandRXYPseudo(MI, SystemZ::L, SystemZ::LFH);1682    return true;1683 1684  case SystemZ::LOCMux:1685    expandLOCPseudo(MI, SystemZ::LOC, SystemZ::LOCFH);1686    return true;1687 1688  case SystemZ::LOCHIMux:1689    expandLOCPseudo(MI, SystemZ::LOCHI, SystemZ::LOCHHI);1690    return true;1691 1692  case SystemZ::STCMux:1693    expandRXYPseudo(MI, SystemZ::STC, SystemZ::STCH);1694    return true;1695 1696  case SystemZ::STHMux:1697    expandRXYPseudo(MI, SystemZ::STH, SystemZ::STHH);1698    return true;1699 1700  case SystemZ::STMux:1701    expandRXYPseudo(MI, SystemZ::ST, SystemZ::STFH);1702    return true;1703 1704  case SystemZ::STOCMux:1705    expandLOCPseudo(MI, SystemZ::STOC, SystemZ::STOCFH);1706    return true;1707 1708  case SystemZ::LHIMux:1709    expandRIPseudo(MI, SystemZ::LHI, SystemZ::IIHF, true);1710    return true;1711 1712  case SystemZ::IIFMux:1713    expandRIPseudo(MI, SystemZ::IILF, SystemZ::IIHF, false);1714    return true;1715 1716  case SystemZ::IILMux:1717    expandRIPseudo(MI, SystemZ::IILL, SystemZ::IIHL, false);1718    return true;1719 1720  case SystemZ::IIHMux:1721    expandRIPseudo(MI, SystemZ::IILH, SystemZ::IIHH, false);1722    return true;1723 1724  case SystemZ::NIFMux:1725    expandRIPseudo(MI, SystemZ::NILF, SystemZ::NIHF, false);1726    return true;1727 1728  case SystemZ::NILMux:1729    expandRIPseudo(MI, SystemZ::NILL, SystemZ::NIHL, false);1730    return true;1731 1732  case SystemZ::NIHMux:1733    expandRIPseudo(MI, SystemZ::NILH, SystemZ::NIHH, false);1734    return true;1735 1736  case SystemZ::OIFMux:1737    expandRIPseudo(MI, SystemZ::OILF, SystemZ::OIHF, false);1738    return true;1739 1740  case SystemZ::OILMux:1741    expandRIPseudo(MI, SystemZ::OILL, SystemZ::OIHL, false);1742    return true;1743 1744  case SystemZ::OIHMux:1745    expandRIPseudo(MI, SystemZ::OILH, SystemZ::OIHH, false);1746    return true;1747 1748  case SystemZ::XIFMux:1749    expandRIPseudo(MI, SystemZ::XILF, SystemZ::XIHF, false);1750    return true;1751 1752  case SystemZ::TMLMux:1753    expandRIPseudo(MI, SystemZ::TMLL, SystemZ::TMHL, false);1754    return true;1755 1756  case SystemZ::TMHMux:1757    expandRIPseudo(MI, SystemZ::TMLH, SystemZ::TMHH, false);1758    return true;1759 1760  case SystemZ::AHIMux:1761    expandRIPseudo(MI, SystemZ::AHI, SystemZ::AIH, false);1762    return true;1763 1764  case SystemZ::AHIMuxK:1765    expandRIEPseudo(MI, SystemZ::AHI, SystemZ::AHIK, SystemZ::AIH);1766    return true;1767 1768  case SystemZ::AFIMux:1769    expandRIPseudo(MI, SystemZ::AFI, SystemZ::AIH, false);1770    return true;1771 1772  case SystemZ::CHIMux:1773    expandRIPseudo(MI, SystemZ::CHI, SystemZ::CIH, false);1774    return true;1775 1776  case SystemZ::CFIMux:1777    expandRIPseudo(MI, SystemZ::CFI, SystemZ::CIH, false);1778    return true;1779 1780  case SystemZ::CLFIMux:1781    expandRIPseudo(MI, SystemZ::CLFI, SystemZ::CLIH, false);1782    return true;1783 1784  case SystemZ::CMux:1785    expandRXYPseudo(MI, SystemZ::C, SystemZ::CHF);1786    return true;1787 1788  case SystemZ::CLMux:1789    expandRXYPseudo(MI, SystemZ::CL, SystemZ::CLHF);1790    return true;1791 1792  case SystemZ::RISBMux: {1793    bool DestIsHigh = SystemZ::isHighReg(MI.getOperand(0).getReg());1794    bool SrcIsHigh = SystemZ::isHighReg(MI.getOperand(2).getReg());1795    if (SrcIsHigh == DestIsHigh)1796      MI.setDesc(get(DestIsHigh ? SystemZ::RISBHH : SystemZ::RISBLL));1797    else {1798      MI.setDesc(get(DestIsHigh ? SystemZ::RISBHL : SystemZ::RISBLH));1799      MI.getOperand(5).setImm(MI.getOperand(5).getImm() ^ 32);1800    }1801    return true;1802  }1803 1804  case SystemZ::ADJDYNALLOC:1805    splitAdjDynAlloc(MI);1806    return true;1807 1808  case TargetOpcode::LOAD_STACK_GUARD:1809    expandLoadStackGuard(&MI);1810    return true;1811 1812  default:1813    return false;1814  }1815}1816 1817unsigned SystemZInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {1818  if (MI.isInlineAsm()) {1819    const MachineFunction *MF = MI.getParent()->getParent();1820    const char *AsmStr = MI.getOperand(0).getSymbolName();1821    return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());1822  }1823  else if (MI.getOpcode() == SystemZ::PATCHPOINT)1824    return PatchPointOpers(&MI).getNumPatchBytes();1825  else if (MI.getOpcode() == SystemZ::STACKMAP)1826    return MI.getOperand(1).getImm();1827  else if (MI.getOpcode() == SystemZ::FENTRY_CALL)1828    return 6;1829  if (MI.getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER)1830    return 18;1831  if (MI.getOpcode() == TargetOpcode::PATCHABLE_RET)1832    return 18 + (MI.getOperand(0).getImm() == SystemZ::CondReturn ? 4 : 0);1833 1834  return MI.getDesc().getSize();1835}1836 1837SystemZII::Branch1838SystemZInstrInfo::getBranchInfo(const MachineInstr &MI) const {1839  switch (MI.getOpcode()) {1840  case SystemZ::BR:1841  case SystemZ::BI:1842  case SystemZ::J:1843  case SystemZ::JG:1844    return SystemZII::Branch(SystemZII::BranchNormal, SystemZ::CCMASK_ANY,1845                             SystemZ::CCMASK_ANY, &MI.getOperand(0));1846 1847  case SystemZ::BRC:1848  case SystemZ::BRCL:1849    return SystemZII::Branch(SystemZII::BranchNormal, MI.getOperand(0).getImm(),1850                             MI.getOperand(1).getImm(), &MI.getOperand(2));1851 1852  case SystemZ::BRCT:1853  case SystemZ::BRCTH:1854    return SystemZII::Branch(SystemZII::BranchCT, SystemZ::CCMASK_ICMP,1855                             SystemZ::CCMASK_CMP_NE, &MI.getOperand(2));1856 1857  case SystemZ::BRCTG:1858    return SystemZII::Branch(SystemZII::BranchCTG, SystemZ::CCMASK_ICMP,1859                             SystemZ::CCMASK_CMP_NE, &MI.getOperand(2));1860 1861  case SystemZ::CIJ:1862  case SystemZ::CRJ:1863    return SystemZII::Branch(SystemZII::BranchC, SystemZ::CCMASK_ICMP,1864                             MI.getOperand(2).getImm(), &MI.getOperand(3));1865 1866  case SystemZ::CLIJ:1867  case SystemZ::CLRJ:1868    return SystemZII::Branch(SystemZII::BranchCL, SystemZ::CCMASK_ICMP,1869                             MI.getOperand(2).getImm(), &MI.getOperand(3));1870 1871  case SystemZ::CGIJ:1872  case SystemZ::CGRJ:1873    return SystemZII::Branch(SystemZII::BranchCG, SystemZ::CCMASK_ICMP,1874                             MI.getOperand(2).getImm(), &MI.getOperand(3));1875 1876  case SystemZ::CLGIJ:1877  case SystemZ::CLGRJ:1878    return SystemZII::Branch(SystemZII::BranchCLG, SystemZ::CCMASK_ICMP,1879                             MI.getOperand(2).getImm(), &MI.getOperand(3));1880 1881  case SystemZ::INLINEASM_BR:1882    // Don't try to analyze asm goto, so pass nullptr as branch target argument.1883    return SystemZII::Branch(SystemZII::AsmGoto, 0, 0, nullptr);1884 1885  default:1886    llvm_unreachable("Unrecognized branch opcode");1887  }1888}1889 1890void SystemZInstrInfo::getLoadStoreOpcodes(const TargetRegisterClass *RC,1891                                           unsigned &LoadOpcode,1892                                           unsigned &StoreOpcode) const {1893  if (RC == &SystemZ::GR32BitRegClass || RC == &SystemZ::ADDR32BitRegClass) {1894    LoadOpcode = SystemZ::L;1895    StoreOpcode = SystemZ::ST;1896  } else if (RC == &SystemZ::GRH32BitRegClass) {1897    LoadOpcode = SystemZ::LFH;1898    StoreOpcode = SystemZ::STFH;1899  } else if (RC == &SystemZ::GRX32BitRegClass) {1900    LoadOpcode = SystemZ::LMux;1901    StoreOpcode = SystemZ::STMux;1902  } else if (RC == &SystemZ::GR64BitRegClass ||1903             RC == &SystemZ::ADDR64BitRegClass) {1904    LoadOpcode = SystemZ::LG;1905    StoreOpcode = SystemZ::STG;1906  } else if (RC == &SystemZ::GR128BitRegClass ||1907             RC == &SystemZ::ADDR128BitRegClass) {1908    LoadOpcode = SystemZ::L128;1909    StoreOpcode = SystemZ::ST128;1910  } else if (RC == &SystemZ::FP16BitRegClass && !STI.hasVector()) {1911    LoadOpcode = SystemZ::LE16;1912    StoreOpcode = SystemZ::STE16;1913  } else if (RC == &SystemZ::FP32BitRegClass) {1914    LoadOpcode = SystemZ::LE;1915    StoreOpcode = SystemZ::STE;1916  } else if (RC == &SystemZ::FP64BitRegClass) {1917    LoadOpcode = SystemZ::LD;1918    StoreOpcode = SystemZ::STD;1919  } else if (RC == &SystemZ::FP128BitRegClass) {1920    LoadOpcode = SystemZ::LX;1921    StoreOpcode = SystemZ::STX;1922  } else if (RC == &SystemZ::FP16BitRegClass ||1923             RC == &SystemZ::VR16BitRegClass) {1924    LoadOpcode = SystemZ::VL16;1925    StoreOpcode = SystemZ::VST16;1926  } else if (RC == &SystemZ::VR32BitRegClass) {1927    LoadOpcode = SystemZ::VL32;1928    StoreOpcode = SystemZ::VST32;1929  } else if (RC == &SystemZ::VR64BitRegClass) {1930    LoadOpcode = SystemZ::VL64;1931    StoreOpcode = SystemZ::VST64;1932  } else if (RC == &SystemZ::VF128BitRegClass ||1933             RC == &SystemZ::VR128BitRegClass) {1934    LoadOpcode = SystemZ::VL;1935    StoreOpcode = SystemZ::VST;1936  } else1937    llvm_unreachable("Unsupported regclass to load or store");1938}1939 1940unsigned SystemZInstrInfo::getOpcodeForOffset(unsigned Opcode,1941                                              int64_t Offset,1942                                              const MachineInstr *MI) const {1943  const MCInstrDesc &MCID = get(Opcode);1944  int64_t Offset2 = (MCID.TSFlags & SystemZII::Is128Bit ? Offset + 8 : Offset);1945  if (isUInt<12>(Offset) && isUInt<12>(Offset2)) {1946    // Get the instruction to use for unsigned 12-bit displacements.1947    int Disp12Opcode = SystemZ::getDisp12Opcode(Opcode);1948    if (Disp12Opcode >= 0)1949      return Disp12Opcode;1950 1951    // All address-related instructions can use unsigned 12-bit1952    // displacements.1953    return Opcode;1954  }1955  if (isInt<20>(Offset) && isInt<20>(Offset2)) {1956    // Get the instruction to use for signed 20-bit displacements.1957    int Disp20Opcode = SystemZ::getDisp20Opcode(Opcode);1958    if (Disp20Opcode >= 0)1959      return Disp20Opcode;1960 1961    // Check whether Opcode allows signed 20-bit displacements.1962    if (MCID.TSFlags & SystemZII::Has20BitOffset)1963      return Opcode;1964 1965    // If a VR32/VR64 reg ended up in an FP register, use the FP opcode.1966    if (MI && MI->getOperand(0).isReg()) {1967      Register Reg = MI->getOperand(0).getReg();1968      if (Reg.isPhysical() && SystemZMC::getFirstReg(Reg) < 16) {1969        switch (Opcode) {1970        case SystemZ::VL32:1971          return SystemZ::LEY;1972        case SystemZ::VST32:1973          return SystemZ::STEY;1974        case SystemZ::VL64:1975          return SystemZ::LDY;1976        case SystemZ::VST64:1977          return SystemZ::STDY;1978        default: break;1979        }1980      }1981    }1982  }1983  return 0;1984}1985 1986bool SystemZInstrInfo::hasDisplacementPairInsn(unsigned Opcode) const {1987  const MCInstrDesc &MCID = get(Opcode);1988  if (MCID.TSFlags & SystemZII::Has20BitOffset)1989    return SystemZ::getDisp12Opcode(Opcode) >= 0;1990  return SystemZ::getDisp20Opcode(Opcode) >= 0;1991}1992 1993unsigned SystemZInstrInfo::getLoadAndTest(unsigned Opcode) const {1994  switch (Opcode) {1995  case SystemZ::L:      return SystemZ::LT;1996  case SystemZ::LY:     return SystemZ::LT;1997  case SystemZ::LG:     return SystemZ::LTG;1998  case SystemZ::LGF:    return SystemZ::LTGF;1999  case SystemZ::LR:     return SystemZ::LTR;2000  case SystemZ::LGFR:   return SystemZ::LTGFR;2001  case SystemZ::LGR:    return SystemZ::LTGR;2002  case SystemZ::LCDFR:  return SystemZ::LCDBR;2003  case SystemZ::LPDFR:  return SystemZ::LPDBR;2004  case SystemZ::LNDFR:  return SystemZ::LNDBR;2005  case SystemZ::LCDFR_32:  return SystemZ::LCEBR;2006  case SystemZ::LPDFR_32:  return SystemZ::LPEBR;2007  case SystemZ::LNDFR_32:  return SystemZ::LNEBR;2008  // On zEC12 we prefer to use RISBGN.  But if there is a chance to2009  // actually use the condition code, we may turn it back into RISGB.2010  // Note that RISBG is not really a "load-and-test" instruction,2011  // but sets the same condition code values, so is OK to use here.2012  case SystemZ::RISBGN: return SystemZ::RISBG;2013  default:              return 0;2014  }2015}2016 2017bool SystemZInstrInfo::isRxSBGMask(uint64_t Mask, unsigned BitSize,2018                                   unsigned &Start, unsigned &End) const {2019  // Reject trivial all-zero masks.2020  Mask &= allOnes(BitSize);2021  if (Mask == 0)2022    return false;2023 2024  // Handle the 1+0+ or 0+1+0* cases.  Start then specifies the index of2025  // the msb and End specifies the index of the lsb.2026  unsigned LSB, Length;2027  if (isShiftedMask_64(Mask, LSB, Length)) {2028    Start = 63 - (LSB + Length - 1);2029    End = 63 - LSB;2030    return true;2031  }2032 2033  // Handle the wrap-around 1+0+1+ cases.  Start then specifies the msb2034  // of the low 1s and End specifies the lsb of the high 1s.2035  if (isShiftedMask_64(Mask ^ allOnes(BitSize), LSB, Length)) {2036    assert(LSB > 0 && "Bottom bit must be set");2037    assert(LSB + Length < BitSize && "Top bit must be set");2038    Start = 63 - (LSB - 1);2039    End = 63 - (LSB + Length);2040    return true;2041  }2042 2043  return false;2044}2045 2046unsigned SystemZInstrInfo::getFusedCompare(unsigned Opcode,2047                                           SystemZII::FusedCompareType Type,2048                                           const MachineInstr *MI) const {2049  switch (Opcode) {2050  case SystemZ::CHI:2051  case SystemZ::CGHI:2052    if (!(MI && isInt<8>(MI->getOperand(1).getImm())))2053      return 0;2054    break;2055  case SystemZ::CLFI:2056  case SystemZ::CLGFI:2057    if (!(MI && isUInt<8>(MI->getOperand(1).getImm())))2058      return 0;2059    break;2060  case SystemZ::CL:2061  case SystemZ::CLG:2062    if (!STI.hasMiscellaneousExtensions())2063      return 0;2064    if (!(MI && MI->getOperand(3).getReg() == 0))2065      return 0;2066    break;2067  }2068  switch (Type) {2069  case SystemZII::CompareAndBranch:2070    switch (Opcode) {2071    case SystemZ::CR:2072      return SystemZ::CRJ;2073    case SystemZ::CGR:2074      return SystemZ::CGRJ;2075    case SystemZ::CHI:2076      return SystemZ::CIJ;2077    case SystemZ::CGHI:2078      return SystemZ::CGIJ;2079    case SystemZ::CLR:2080      return SystemZ::CLRJ;2081    case SystemZ::CLGR:2082      return SystemZ::CLGRJ;2083    case SystemZ::CLFI:2084      return SystemZ::CLIJ;2085    case SystemZ::CLGFI:2086      return SystemZ::CLGIJ;2087    default:2088      return 0;2089    }2090  case SystemZII::CompareAndReturn:2091    switch (Opcode) {2092    case SystemZ::CR:2093      return SystemZ::CRBReturn;2094    case SystemZ::CGR:2095      return SystemZ::CGRBReturn;2096    case SystemZ::CHI:2097      return SystemZ::CIBReturn;2098    case SystemZ::CGHI:2099      return SystemZ::CGIBReturn;2100    case SystemZ::CLR:2101      return SystemZ::CLRBReturn;2102    case SystemZ::CLGR:2103      return SystemZ::CLGRBReturn;2104    case SystemZ::CLFI:2105      return SystemZ::CLIBReturn;2106    case SystemZ::CLGFI:2107      return SystemZ::CLGIBReturn;2108    default:2109      return 0;2110    }2111  case SystemZII::CompareAndSibcall:2112    switch (Opcode) {2113    case SystemZ::CR:2114      return SystemZ::CRBCall;2115    case SystemZ::CGR:2116      return SystemZ::CGRBCall;2117    case SystemZ::CHI:2118      return SystemZ::CIBCall;2119    case SystemZ::CGHI:2120      return SystemZ::CGIBCall;2121    case SystemZ::CLR:2122      return SystemZ::CLRBCall;2123    case SystemZ::CLGR:2124      return SystemZ::CLGRBCall;2125    case SystemZ::CLFI:2126      return SystemZ::CLIBCall;2127    case SystemZ::CLGFI:2128      return SystemZ::CLGIBCall;2129    default:2130      return 0;2131    }2132  case SystemZII::CompareAndTrap:2133    switch (Opcode) {2134    case SystemZ::CR:2135      return SystemZ::CRT;2136    case SystemZ::CGR:2137      return SystemZ::CGRT;2138    case SystemZ::CHI:2139      return SystemZ::CIT;2140    case SystemZ::CGHI:2141      return SystemZ::CGIT;2142    case SystemZ::CLR:2143      return SystemZ::CLRT;2144    case SystemZ::CLGR:2145      return SystemZ::CLGRT;2146    case SystemZ::CLFI:2147      return SystemZ::CLFIT;2148    case SystemZ::CLGFI:2149      return SystemZ::CLGIT;2150    case SystemZ::CL:2151      return SystemZ::CLT;2152    case SystemZ::CLG:2153      return SystemZ::CLGT;2154    default:2155      return 0;2156    }2157  }2158  return 0;2159}2160 2161bool SystemZInstrInfo::2162prepareCompareSwapOperands(MachineBasicBlock::iterator const MBBI) const {2163  assert(MBBI->isCompare() && MBBI->getOperand(0).isReg() &&2164         MBBI->getOperand(1).isReg() && !MBBI->mayLoad() &&2165         "Not a compare reg/reg.");2166 2167  MachineBasicBlock *MBB = MBBI->getParent();2168  bool CCLive = true;2169  SmallVector<MachineInstr *, 4> CCUsers;2170  for (MachineInstr &MI : llvm::make_range(std::next(MBBI), MBB->end())) {2171    if (MI.readsRegister(SystemZ::CC, /*TRI=*/nullptr)) {2172      unsigned Flags = MI.getDesc().TSFlags;2173      if ((Flags & SystemZII::CCMaskFirst) || (Flags & SystemZII::CCMaskLast))2174        CCUsers.push_back(&MI);2175      else2176        return false;2177    }2178    if (MI.definesRegister(SystemZ::CC, /*TRI=*/nullptr)) {2179      CCLive = false;2180      break;2181    }2182  }2183  if (CCLive) {2184    LiveRegUnits LiveRegs(*MBB->getParent()->getSubtarget().getRegisterInfo());2185    LiveRegs.addLiveOuts(*MBB);2186    if (!LiveRegs.available(SystemZ::CC))2187      return false;2188  }2189 2190  // Update all CC users.2191  for (unsigned Idx = 0; Idx < CCUsers.size(); ++Idx) {2192    unsigned Flags = CCUsers[Idx]->getDesc().TSFlags;2193    unsigned FirstOpNum = ((Flags & SystemZII::CCMaskFirst) ?2194                           0 : CCUsers[Idx]->getNumExplicitOperands() - 2);2195    MachineOperand &CCMaskMO = CCUsers[Idx]->getOperand(FirstOpNum + 1);2196    unsigned NewCCMask = SystemZ::reverseCCMask(CCMaskMO.getImm());2197    CCMaskMO.setImm(NewCCMask);2198  }2199 2200  return true;2201}2202 2203unsigned SystemZ::reverseCCMask(unsigned CCMask) {2204  return ((CCMask & SystemZ::CCMASK_CMP_EQ) |2205          ((CCMask & SystemZ::CCMASK_CMP_GT) ? SystemZ::CCMASK_CMP_LT : 0) |2206          ((CCMask & SystemZ::CCMASK_CMP_LT) ? SystemZ::CCMASK_CMP_GT : 0) |2207          (CCMask & SystemZ::CCMASK_CMP_UO));2208}2209 2210MachineBasicBlock *SystemZ::emitBlockAfter(MachineBasicBlock *MBB) {2211  MachineFunction &MF = *MBB->getParent();2212  MachineBasicBlock *NewMBB = MF.CreateMachineBasicBlock(MBB->getBasicBlock());2213  MF.insert(std::next(MachineFunction::iterator(MBB)), NewMBB);2214  return NewMBB;2215}2216 2217MachineBasicBlock *SystemZ::splitBlockAfter(MachineBasicBlock::iterator MI,2218                                            MachineBasicBlock *MBB) {2219  MachineBasicBlock *NewMBB = emitBlockAfter(MBB);2220  NewMBB->splice(NewMBB->begin(), MBB,2221                 std::next(MachineBasicBlock::iterator(MI)), MBB->end());2222  NewMBB->transferSuccessorsAndUpdatePHIs(MBB);2223  return NewMBB;2224}2225 2226MachineBasicBlock *SystemZ::splitBlockBefore(MachineBasicBlock::iterator MI,2227                                             MachineBasicBlock *MBB) {2228  MachineBasicBlock *NewMBB = emitBlockAfter(MBB);2229  NewMBB->splice(NewMBB->begin(), MBB, MI, MBB->end());2230  NewMBB->transferSuccessorsAndUpdatePHIs(MBB);2231  return NewMBB;2232}2233 2234unsigned SystemZInstrInfo::getLoadAndTrap(unsigned Opcode) const {2235  if (!STI.hasLoadAndTrap())2236    return 0;2237  switch (Opcode) {2238  case SystemZ::L:2239  case SystemZ::LY:2240    return SystemZ::LAT;2241  case SystemZ::LG:2242    return SystemZ::LGAT;2243  case SystemZ::LFH:2244    return SystemZ::LFHAT;2245  case SystemZ::LLGF:2246    return SystemZ::LLGFAT;2247  case SystemZ::LLGT:2248    return SystemZ::LLGTAT;2249  }2250  return 0;2251}2252 2253void SystemZInstrInfo::loadImmediate(MachineBasicBlock &MBB,2254                                     MachineBasicBlock::iterator MBBI,2255                                     unsigned Reg, uint64_t Value) const {2256  DebugLoc DL = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();2257  unsigned Opcode = 0;2258  if (isInt<16>(Value))2259    Opcode = SystemZ::LGHI;2260  else if (SystemZ::isImmLL(Value))2261    Opcode = SystemZ::LLILL;2262  else if (SystemZ::isImmLH(Value)) {2263    Opcode = SystemZ::LLILH;2264    Value >>= 16;2265  }2266  else if (isInt<32>(Value))2267    Opcode = SystemZ::LGFI;2268  if (Opcode) {2269    BuildMI(MBB, MBBI, DL, get(Opcode), Reg).addImm(Value);2270    return;2271  }2272 2273  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();2274  assert (MRI.isSSA() &&  "Huge values only handled before reg-alloc .");2275  Register Reg0 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);2276  Register Reg1 = MRI.createVirtualRegister(&SystemZ::GR64BitRegClass);2277  BuildMI(MBB, MBBI, DL, get(SystemZ::IMPLICIT_DEF), Reg0);2278  BuildMI(MBB, MBBI, DL, get(SystemZ::IIHF64), Reg1)2279    .addReg(Reg0).addImm(Value >> 32);2280  BuildMI(MBB, MBBI, DL, get(SystemZ::IILF64), Reg)2281    .addReg(Reg1).addImm(Value & ((uint64_t(1) << 32) - 1));2282}2283 2284bool SystemZInstrInfo::verifyInstruction(const MachineInstr &MI,2285                                         StringRef &ErrInfo) const {2286  const MCInstrDesc &MCID = MI.getDesc();2287  for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {2288    if (I >= MCID.getNumOperands())2289      break;2290    const MachineOperand &Op = MI.getOperand(I);2291    const MCOperandInfo &MCOI = MCID.operands()[I];2292    // Addressing modes have register and immediate operands. Op should be a2293    // register (or frame index) operand if MCOI.RegClass contains a valid2294    // register class, or an immediate otherwise.2295    if (MCOI.OperandType == MCOI::OPERAND_MEMORY &&2296        ((MCOI.RegClass != -1 && !Op.isReg() && !Op.isFI()) ||2297         (MCOI.RegClass == -1 && !Op.isImm()))) {2298      ErrInfo = "Addressing mode operands corrupt!";2299      return false;2300    }2301  }2302 2303  return true;2304}2305 2306bool SystemZInstrInfo::2307areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,2308                                const MachineInstr &MIb) const {2309 2310  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand())2311    return false;2312 2313  // If mem-operands show that the same address Value is used by both2314  // instructions, check for non-overlapping offsets and widths. Not2315  // sure if a register based analysis would be an improvement...2316 2317  MachineMemOperand *MMOa = *MIa.memoperands_begin();2318  MachineMemOperand *MMOb = *MIb.memoperands_begin();2319  const Value *VALa = MMOa->getValue();2320  const Value *VALb = MMOb->getValue();2321  bool SameVal = (VALa && VALb && (VALa == VALb));2322  if (!SameVal) {2323    const PseudoSourceValue *PSVa = MMOa->getPseudoValue();2324    const PseudoSourceValue *PSVb = MMOb->getPseudoValue();2325    if (PSVa && PSVb && (PSVa == PSVb))2326      SameVal = true;2327  }2328  if (SameVal) {2329    int OffsetA = MMOa->getOffset(), OffsetB = MMOb->getOffset();2330    LocationSize WidthA = MMOa->getSize(), WidthB = MMOb->getSize();2331    int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;2332    int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;2333    LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;2334    if (LowWidth.hasValue() &&2335        LowOffset + (int)LowWidth.getValue() <= HighOffset)2336      return true;2337  }2338 2339  return false;2340}2341 2342bool SystemZInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,2343                                               const Register Reg,2344                                               int64_t &ImmVal) const {2345 2346  if (MI.getOpcode() == SystemZ::VGBM && Reg == MI.getOperand(0).getReg()) {2347    ImmVal = MI.getOperand(1).getImm();2348    // TODO: Handle non-0 values2349    return ImmVal == 0;2350  }2351 2352  return false;2353}2354 2355std::optional<DestSourcePair>2356SystemZInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {2357  // if MI is a simple single-register copy operation, return operand pair2358  if (MI.isMoveReg())2359    return DestSourcePair(MI.getOperand(0), MI.getOperand(1));2360 2361  return std::nullopt;2362}2363 2364std::pair<unsigned, unsigned>2365SystemZInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {2366  return std::make_pair(TF, 0u);2367}2368 2369ArrayRef<std::pair<unsigned, const char *>>2370SystemZInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {2371  using namespace SystemZII;2372 2373  static const std::pair<unsigned, const char *> TargetFlags[] = {2374      {MO_ADA_DATA_SYMBOL_ADDR, "systemz-ada-datasymboladdr"},2375      {MO_ADA_INDIRECT_FUNC_DESC, "systemz-ada-indirectfuncdesc"},2376      {MO_ADA_DIRECT_FUNC_DESC, "systemz-ada-directfuncdesc"}};2377  return ArrayRef(TargetFlags);2378}2379