brintos / llvm-project-archived public Read only

1//===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8/// \file9/// This file implements the IRTranslator class.10//===----------------------------------------------------------------------===//11 12#include "llvm/CodeGen/GlobalISel/IRTranslator.h"13#include "llvm/ADT/PostOrderIterator.h"14#include "llvm/ADT/STLExtras.h"15#include "llvm/ADT/ScopeExit.h"16#include "llvm/ADT/SmallVector.h"17#include "llvm/Analysis/AliasAnalysis.h"18#include "llvm/Analysis/AssumptionCache.h"19#include "llvm/Analysis/BranchProbabilityInfo.h"20#include "llvm/Analysis/Loads.h"21#include "llvm/Analysis/OptimizationRemarkEmitter.h"22#include "llvm/Analysis/ValueTracking.h"23#include "llvm/Analysis/VectorUtils.h"24#include "llvm/CodeGen/Analysis.h"25#include "llvm/CodeGen/GlobalISel/CSEInfo.h"26#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"27#include "llvm/CodeGen/GlobalISel/CallLowering.h"28#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"29#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"30#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"31#include "llvm/CodeGen/LowLevelTypeUtils.h"32#include "llvm/CodeGen/MachineBasicBlock.h"33#include "llvm/CodeGen/MachineFrameInfo.h"34#include "llvm/CodeGen/MachineFunction.h"35#include "llvm/CodeGen/MachineInstrBuilder.h"36#include "llvm/CodeGen/MachineMemOperand.h"37#include "llvm/CodeGen/MachineModuleInfo.h"38#include "llvm/CodeGen/MachineOperand.h"39#include "llvm/CodeGen/MachineRegisterInfo.h"40#include "llvm/CodeGen/StackProtector.h"41#include "llvm/CodeGen/SwitchLoweringUtils.h"42#include "llvm/CodeGen/TargetFrameLowering.h"43#include "llvm/CodeGen/TargetInstrInfo.h"44#include "llvm/CodeGen/TargetLowering.h"45#include "llvm/CodeGen/TargetOpcodes.h"46#include "llvm/CodeGen/TargetPassConfig.h"47#include "llvm/CodeGen/TargetRegisterInfo.h"48#include "llvm/CodeGen/TargetSubtargetInfo.h"49#include "llvm/CodeGenTypes/LowLevelType.h"50#include "llvm/IR/BasicBlock.h"51#include "llvm/IR/CFG.h"52#include "llvm/IR/Constant.h"53#include "llvm/IR/Constants.h"54#include "llvm/IR/DataLayout.h"55#include "llvm/IR/DerivedTypes.h"56#include "llvm/IR/DiagnosticInfo.h"57#include "llvm/IR/Function.h"58#include "llvm/IR/GetElementPtrTypeIterator.h"59#include "llvm/IR/InlineAsm.h"60#include "llvm/IR/InstrTypes.h"61#include "llvm/IR/Instructions.h"62#include "llvm/IR/IntrinsicInst.h"63#include "llvm/IR/Intrinsics.h"64#include "llvm/IR/IntrinsicsAMDGPU.h"65#include "llvm/IR/LLVMContext.h"66#include "llvm/IR/Metadata.h"67#include "llvm/IR/PatternMatch.h"68#include "llvm/IR/Statepoint.h"69#include "llvm/IR/Type.h"70#include "llvm/IR/User.h"71#include "llvm/IR/Value.h"72#include "llvm/InitializePasses.h"73#include "llvm/MC/MCContext.h"74#include "llvm/Pass.h"75#include "llvm/Support/Casting.h"76#include "llvm/Support/CodeGen.h"77#include "llvm/Support/Debug.h"78#include "llvm/Support/ErrorHandling.h"79#include "llvm/Support/MathExtras.h"80#include "llvm/Support/raw_ostream.h"81#include "llvm/Target/TargetMachine.h"82#include "llvm/Transforms/Utils/Local.h"83#include "llvm/Transforms/Utils/MemoryOpRemark.h"84#include <algorithm>85#include <cassert>86#include <cstdint>87#include <iterator>88#include <optional>89#include <string>90#include <utility>91#include <vector>92 93#define DEBUG_TYPE "irtranslator"94 95using namespace llvm;96 97static cl::opt<bool>98    EnableCSEInIRTranslator("enable-cse-in-irtranslator",99                            cl::desc("Should enable CSE in irtranslator"),100                            cl::Optional, cl::init(false));101char IRTranslator::ID = 0;102 103INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",104                false, false)105INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)106INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)107INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)108INITIALIZE_PASS_DEPENDENCY(StackProtector)109INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)110INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",111                false, false)112 113static void reportTranslationError(MachineFunction &MF,114                                   const TargetPassConfig &TPC,115                                   OptimizationRemarkEmitter &ORE,116                                   OptimizationRemarkMissed &R) {117  MF.getProperties().setFailedISel();118 119  // Print the function name explicitly if we don't have a debug location (which120  // makes the diagnostic less useful) or if we're going to emit a raw error.121  if (!R.getLocation().isValid() || TPC.isGlobalISelAbortEnabled())122    R << (" (in function: " + MF.getName() + ")").str();123 124  if (TPC.isGlobalISelAbortEnabled())125    report_fatal_error(Twine(R.getMsg()));126  else127    ORE.emit(R);128}129 130IRTranslator::IRTranslator(CodeGenOptLevel optlevel)131    : MachineFunctionPass(ID), OptLevel(optlevel) {}132 133#ifndef NDEBUG134namespace {135/// Verify that every instruction created has the same DILocation as the136/// instruction being translated.137class DILocationVerifier : public GISelChangeObserver {138  const Instruction *CurrInst = nullptr;139 140public:141  DILocationVerifier() = default;142  ~DILocationVerifier() override = default;143 144  const Instruction *getCurrentInst() const { return CurrInst; }145  void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }146 147  void erasingInstr(MachineInstr &MI) override {}148  void changingInstr(MachineInstr &MI) override {}149  void changedInstr(MachineInstr &MI) override {}150 151  void createdInstr(MachineInstr &MI) override {152    assert(getCurrentInst() && "Inserted instruction without a current MI");153 154    // Only print the check message if we're actually checking it.155#ifndef NDEBUG156    LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst157                      << " was copied to " << MI);158#endif159    // We allow insts in the entry block to have no debug loc because160    // they could have originated from constants, and we don't want a jumpy161    // debug experience.162    assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||163            (MI.getParent()->isEntryBlock() && !MI.getDebugLoc()) ||164            (MI.isDebugInstr())) &&165           "Line info was not transferred to all instructions");166  }167};168} // namespace169#endif // ifndef NDEBUG170 171 172void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {173  AU.addRequired<StackProtector>();174  AU.addRequired<TargetPassConfig>();175  AU.addRequired<GISelCSEAnalysisWrapperPass>();176  AU.addRequired<AssumptionCacheTracker>();177  if (OptLevel != CodeGenOptLevel::None) {178    AU.addRequired<BranchProbabilityInfoWrapperPass>();179    AU.addRequired<AAResultsWrapperPass>();180  }181  AU.addRequired<TargetLibraryInfoWrapperPass>();182  AU.addPreserved<TargetLibraryInfoWrapperPass>();183  getSelectionDAGFallbackAnalysisUsage(AU);184  MachineFunctionPass::getAnalysisUsage(AU);185}186 187IRTranslator::ValueToVRegInfo::VRegListT &188IRTranslator::allocateVRegs(const Value &Val) {189  auto VRegsIt = VMap.findVRegs(Val);190  if (VRegsIt != VMap.vregs_end())191    return *VRegsIt->second;192  auto *Regs = VMap.getVRegs(Val);193  auto *Offsets = VMap.getOffsets(Val);194  SmallVector<LLT, 4> SplitTys;195  computeValueLLTs(*DL, *Val.getType(), SplitTys,196                   Offsets->empty() ? Offsets : nullptr);197  for (unsigned i = 0; i < SplitTys.size(); ++i)198    Regs->push_back(0);199  return *Regs;200}201 202ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {203  auto VRegsIt = VMap.findVRegs(Val);204  if (VRegsIt != VMap.vregs_end())205    return *VRegsIt->second;206 207  if (Val.getType()->isVoidTy())208    return *VMap.getVRegs(Val);209 210  // Create entry for this type.211  auto *VRegs = VMap.getVRegs(Val);212  auto *Offsets = VMap.getOffsets(Val);213 214  if (!Val.getType()->isTokenTy())215    assert(Val.getType()->isSized() &&216           "Don't know how to create an empty vreg");217 218  SmallVector<LLT, 4> SplitTys;219  computeValueLLTs(*DL, *Val.getType(), SplitTys,220                   Offsets->empty() ? Offsets : nullptr);221 222  if (!isa<Constant>(Val)) {223    for (auto Ty : SplitTys)224      VRegs->push_back(MRI->createGenericVirtualRegister(Ty));225    return *VRegs;226  }227 228  if (Val.getType()->isAggregateType()) {229    // UndefValue, ConstantAggregateZero230    auto &C = cast<Constant>(Val);231    unsigned Idx = 0;232    while (auto Elt = C.getAggregateElement(Idx++)) {233      auto EltRegs = getOrCreateVRegs(*Elt);234      llvm::append_range(*VRegs, EltRegs);235    }236  } else {237    assert(SplitTys.size() == 1 && "unexpectedly split LLT");238    VRegs->push_back(MRI->createGenericVirtualRegister(SplitTys[0]));239    bool Success = translate(cast<Constant>(Val), VRegs->front());240    if (!Success) {241      OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",242                                 MF->getFunction().getSubprogram(),243                                 &MF->getFunction().getEntryBlock());244      R << "unable to translate constant: " << ore::NV("Type", Val.getType());245      reportTranslationError(*MF, *TPC, *ORE, R);246      return *VRegs;247    }248  }249 250  return *VRegs;251}252 253int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) {254  auto [MapEntry, Inserted] = FrameIndices.try_emplace(&AI);255  if (!Inserted)256    return MapEntry->second;257 258  uint64_t ElementSize = DL->getTypeAllocSize(AI.getAllocatedType());259  uint64_t Size =260      ElementSize * cast<ConstantInt>(AI.getArraySize())->getZExtValue();261 262  // Always allocate at least one byte.263  Size = std::max<uint64_t>(Size, 1u);264 265  int &FI = MapEntry->second;266  FI = MF->getFrameInfo().CreateStackObject(Size, AI.getAlign(), false, &AI);267  return FI;268}269 270Align IRTranslator::getMemOpAlign(const Instruction &I) {271  if (const StoreInst *SI = dyn_cast<StoreInst>(&I))272    return SI->getAlign();273  if (const LoadInst *LI = dyn_cast<LoadInst>(&I))274    return LI->getAlign();275  if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I))276    return AI->getAlign();277  if (const AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(&I))278    return AI->getAlign();279 280  OptimizationRemarkMissed R("gisel-irtranslator", "", &I);281  R << "unable to translate memop: " << ore::NV("Opcode", &I);282  reportTranslationError(*MF, *TPC, *ORE, R);283  return Align(1);284}285 286MachineBasicBlock &IRTranslator::getMBB(const BasicBlock &BB) {287  MachineBasicBlock *MBB = FuncInfo.getMBB(&BB);288  assert(MBB && "BasicBlock was not encountered before");289  return *MBB;290}291 292void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) {293  assert(NewPred && "new predecessor must be a real MachineBasicBlock");294  MachinePreds[Edge].push_back(NewPred);295}296 297static bool targetSupportsBF16Type(const MachineFunction *MF) {298  return MF->getTarget().getTargetTriple().isSPIRV();299}300 301static bool containsBF16Type(const User &U) {302  // BF16 cannot currently be represented by LLT, to avoid miscompiles we303  // prevent any instructions using them. FIXME: This can be removed once LLT304  // supports bfloat.305  return U.getType()->getScalarType()->isBFloatTy() ||306         any_of(U.operands(), [](Value *V) {307           return V->getType()->getScalarType()->isBFloatTy();308         });309}310 311bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,312                                     MachineIRBuilder &MIRBuilder) {313  if (containsBF16Type(U) && !targetSupportsBF16Type(MF))314    return false;315 316  // Get or create a virtual register for each value.317  // Unless the value is a Constant => loadimm cst?318  // or inline constant each time?319  // Creation of a virtual register needs to have a size.320  Register Op0 = getOrCreateVReg(*U.getOperand(0));321  Register Op1 = getOrCreateVReg(*U.getOperand(1));322  Register Res = getOrCreateVReg(U);323  uint32_t Flags = 0;324  if (isa<Instruction>(U)) {325    const Instruction &I = cast<Instruction>(U);326    Flags = MachineInstr::copyFlagsFromInstruction(I);327  }328 329  MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);330  return true;331}332 333bool IRTranslator::translateUnaryOp(unsigned Opcode, const User &U,334                                    MachineIRBuilder &MIRBuilder) {335  if (containsBF16Type(U) && !targetSupportsBF16Type(MF))336    return false;337 338  Register Op0 = getOrCreateVReg(*U.getOperand(0));339  Register Res = getOrCreateVReg(U);340  uint32_t Flags = 0;341  if (isa<Instruction>(U)) {342    const Instruction &I = cast<Instruction>(U);343    Flags = MachineInstr::copyFlagsFromInstruction(I);344  }345  MIRBuilder.buildInstr(Opcode, {Res}, {Op0}, Flags);346  return true;347}348 349bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {350  return translateUnaryOp(TargetOpcode::G_FNEG, U, MIRBuilder);351}352 353bool IRTranslator::translateCompare(const User &U,354                                    MachineIRBuilder &MIRBuilder) {355  if (containsBF16Type(U) && !targetSupportsBF16Type(MF))356    return false;357 358  auto *CI = cast<CmpInst>(&U);359  Register Op0 = getOrCreateVReg(*U.getOperand(0));360  Register Op1 = getOrCreateVReg(*U.getOperand(1));361  Register Res = getOrCreateVReg(U);362  CmpInst::Predicate Pred = CI->getPredicate();363  uint32_t Flags = MachineInstr::copyFlagsFromInstruction(*CI);364  if (CmpInst::isIntPredicate(Pred))365    MIRBuilder.buildICmp(Pred, Res, Op0, Op1, Flags);366  else if (Pred == CmpInst::FCMP_FALSE)367    MIRBuilder.buildCopy(368        Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));369  else if (Pred == CmpInst::FCMP_TRUE)370    MIRBuilder.buildCopy(371        Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));372  else373    MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);374 375  return true;376}377 378bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) {379  const ReturnInst &RI = cast<ReturnInst>(U);380  const Value *Ret = RI.getReturnValue();381  if (Ret && DL->getTypeStoreSize(Ret->getType()).isZero())382    Ret = nullptr;383 384  ArrayRef<Register> VRegs;385  if (Ret)386    VRegs = getOrCreateVRegs(*Ret);387 388  Register SwiftErrorVReg = 0;389  if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) {390    SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt(391        &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg());392  }393 394  // The target may mess up with the insertion point, but395  // this is not important as a return is the last instruction396  // of the block anyway.397  return CLI->lowerReturn(MIRBuilder, Ret, VRegs, FuncInfo, SwiftErrorVReg);398}399 400void IRTranslator::emitBranchForMergedCondition(401    const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,402    MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,403    BranchProbability TProb, BranchProbability FProb, bool InvertCond) {404  // If the leaf of the tree is a comparison, merge the condition into405  // the caseblock.406  if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {407    CmpInst::Predicate Condition;408    if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {409      Condition = InvertCond ? IC->getInversePredicate() : IC->getPredicate();410    } else {411      const FCmpInst *FC = cast<FCmpInst>(Cond);412      Condition = InvertCond ? FC->getInversePredicate() : FC->getPredicate();413    }414 415    SwitchCG::CaseBlock CB(Condition, false, BOp->getOperand(0),416                           BOp->getOperand(1), nullptr, TBB, FBB, CurBB,417                           CurBuilder->getDebugLoc(), TProb, FProb);418    SL->SwitchCases.push_back(CB);419    return;420  }421 422  // Create a CaseBlock record representing this branch.423  CmpInst::Predicate Pred = InvertCond ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;424  SwitchCG::CaseBlock CB(425      Pred, false, Cond, ConstantInt::getTrue(MF->getFunction().getContext()),426      nullptr, TBB, FBB, CurBB, CurBuilder->getDebugLoc(), TProb, FProb);427  SL->SwitchCases.push_back(CB);428}429 430static bool isValInBlock(const Value *V, const BasicBlock *BB) {431  if (const Instruction *I = dyn_cast<Instruction>(V))432    return I->getParent() == BB;433  return true;434}435 436void IRTranslator::findMergedConditions(437    const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB,438    MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB,439    Instruction::BinaryOps Opc, BranchProbability TProb,440    BranchProbability FProb, bool InvertCond) {441  using namespace PatternMatch;442  assert((Opc == Instruction::And || Opc == Instruction::Or) &&443         "Expected Opc to be AND/OR");444  // Skip over not part of the tree and remember to invert op and operands at445  // next level.446  Value *NotCond;447  if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) &&448      isValInBlock(NotCond, CurBB->getBasicBlock())) {449    findMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,450                         !InvertCond);451    return;452  }453 454  const Instruction *BOp = dyn_cast<Instruction>(Cond);455  const Value *BOpOp0, *BOpOp1;456  // Compute the effective opcode for Cond, taking into account whether it needs457  // to be inverted, e.g.458  //   and (not (or A, B)), C459  // gets lowered as460  //   and (and (not A, not B), C)461  Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0;462  if (BOp) {463    BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1)))464               ? Instruction::And465               : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1)))466                      ? Instruction::Or467                      : (Instruction::BinaryOps)0);468    if (InvertCond) {469      if (BOpc == Instruction::And)470        BOpc = Instruction::Or;471      else if (BOpc == Instruction::Or)472        BOpc = Instruction::And;473    }474  }475 476  // If this node is not part of the or/and tree, emit it as a branch.477  // Note that all nodes in the tree should have same opcode.478  bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse();479  if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() ||480      !isValInBlock(BOpOp0, CurBB->getBasicBlock()) ||481      !isValInBlock(BOpOp1, CurBB->getBasicBlock())) {482    emitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb,483                                 InvertCond);484    return;485  }486 487  //  Create TmpBB after CurBB.488  MachineFunction::iterator BBI(CurBB);489  MachineBasicBlock *TmpBB =490      MF->CreateMachineBasicBlock(CurBB->getBasicBlock());491  CurBB->getParent()->insert(++BBI, TmpBB);492 493  if (Opc == Instruction::Or) {494    // Codegen X | Y as:495    // BB1:496    //   jmp_if_X TBB497    //   jmp TmpBB498    // TmpBB:499    //   jmp_if_Y TBB500    //   jmp FBB501    //502 503    // We have flexibility in setting Prob for BB1 and Prob for TmpBB.504    // The requirement is that505    //   TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)506    //     = TrueProb for original BB.507    // Assuming the original probabilities are A and B, one choice is to set508    // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to509    // A/(1+B) and 2B/(1+B). This choice assumes that510    //   TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.511    // Another choice is to assume TrueProb for BB1 equals to TrueProb for512    // TmpBB, but the math is more complicated.513 514    auto NewTrueProb = TProb / 2;515    auto NewFalseProb = TProb / 2 + FProb;516    // Emit the LHS condition.517    findMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb,518                         NewFalseProb, InvertCond);519 520    // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).521    SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};522    BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());523    // Emit the RHS condition into TmpBB.524    findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],525                         Probs[1], InvertCond);526  } else {527    assert(Opc == Instruction::And && "Unknown merge op!");528    // Codegen X & Y as:529    // BB1:530    //   jmp_if_X TmpBB531    //   jmp FBB532    // TmpBB:533    //   jmp_if_Y TBB534    //   jmp FBB535    //536    //  This requires creation of TmpBB after CurBB.537 538    // We have flexibility in setting Prob for BB1 and Prob for TmpBB.539    // The requirement is that540    //   FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)541    //     = FalseProb for original BB.542    // Assuming the original probabilities are A and B, one choice is to set543    // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to544    // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==545    // TrueProb for BB1 * FalseProb for TmpBB.546 547    auto NewTrueProb = TProb + FProb / 2;548    auto NewFalseProb = FProb / 2;549    // Emit the LHS condition.550    findMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb,551                         NewFalseProb, InvertCond);552 553    // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).554    SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};555    BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());556    // Emit the RHS condition into TmpBB.557    findMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0],558                         Probs[1], InvertCond);559  }560}561 562bool IRTranslator::shouldEmitAsBranches(563    const std::vector<SwitchCG::CaseBlock> &Cases) {564  // For multiple cases, it's better to emit as branches.565  if (Cases.size() != 2)566    return true;567 568  // If this is two comparisons of the same values or'd or and'd together, they569  // will get folded into a single comparison, so don't emit two blocks.570  if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&571       Cases[0].CmpRHS == Cases[1].CmpRHS) ||572      (Cases[0].CmpRHS == Cases[1].CmpLHS &&573       Cases[0].CmpLHS == Cases[1].CmpRHS)) {574    return false;575  }576 577  // Handle: (X != null) | (Y != null) --> (X|Y) != 0578  // Handle: (X == null) & (Y == null) --> (X|Y) == 0579  if (Cases[0].CmpRHS == Cases[1].CmpRHS &&580      Cases[0].PredInfo.Pred == Cases[1].PredInfo.Pred &&581      isa<Constant>(Cases[0].CmpRHS) &&582      cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {583    if (Cases[0].PredInfo.Pred == CmpInst::ICMP_EQ &&584        Cases[0].TrueBB == Cases[1].ThisBB)585      return false;586    if (Cases[0].PredInfo.Pred == CmpInst::ICMP_NE &&587        Cases[0].FalseBB == Cases[1].ThisBB)588      return false;589  }590 591  return true;592}593 594bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) {595  const BranchInst &BrInst = cast<BranchInst>(U);596  auto &CurMBB = MIRBuilder.getMBB();597  auto *Succ0MBB = &getMBB(*BrInst.getSuccessor(0));598 599  if (BrInst.isUnconditional()) {600    // If the unconditional target is the layout successor, fallthrough.601    if (OptLevel == CodeGenOptLevel::None ||602        !CurMBB.isLayoutSuccessor(Succ0MBB))603      MIRBuilder.buildBr(*Succ0MBB);604 605    // Link successors.606    for (const BasicBlock *Succ : successors(&BrInst))607      CurMBB.addSuccessor(&getMBB(*Succ));608    return true;609  }610 611  // If this condition is one of the special cases we handle, do special stuff612  // now.613  const Value *CondVal = BrInst.getCondition();614  MachineBasicBlock *Succ1MBB = &getMBB(*BrInst.getSuccessor(1));615 616  // If this is a series of conditions that are or'd or and'd together, emit617  // this as a sequence of branches instead of setcc's with and/or operations.618  // As long as jumps are not expensive (exceptions for multi-use logic ops,619  // unpredictable branches, and vector extracts because those jumps are likely620  // expensive for any target), this should improve performance.621  // For example, instead of something like:622  //     cmp A, B623  //     C = seteq624  //     cmp D, E625  //     F = setle626  //     or C, F627  //     jnz foo628  // Emit:629  //     cmp A, B630  //     je foo631  //     cmp D, E632  //     jle foo633  using namespace PatternMatch;634  const Instruction *CondI = dyn_cast<Instruction>(CondVal);635  if (!TLI->isJumpExpensive() && CondI && CondI->hasOneUse() &&636      !BrInst.hasMetadata(LLVMContext::MD_unpredictable)) {637    Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0;638    Value *Vec;639    const Value *BOp0, *BOp1;640    if (match(CondI, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1))))641      Opcode = Instruction::And;642    else if (match(CondI, m_LogicalOr(m_Value(BOp0), m_Value(BOp1))))643      Opcode = Instruction::Or;644 645    if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) &&646                    match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) {647      findMergedConditions(CondI, Succ0MBB, Succ1MBB, &CurMBB, &CurMBB, Opcode,648                           getEdgeProbability(&CurMBB, Succ0MBB),649                           getEdgeProbability(&CurMBB, Succ1MBB),650                           /*InvertCond=*/false);651      assert(SL->SwitchCases[0].ThisBB == &CurMBB && "Unexpected lowering!");652 653      // Allow some cases to be rejected.654      if (shouldEmitAsBranches(SL->SwitchCases)) {655        // Emit the branch for this block.656        emitSwitchCase(SL->SwitchCases[0], &CurMBB, *CurBuilder);657        SL->SwitchCases.erase(SL->SwitchCases.begin());658        return true;659      }660 661      // Okay, we decided not to do this, remove any inserted MBB's and clear662      // SwitchCases.663      for (unsigned I = 1, E = SL->SwitchCases.size(); I != E; ++I)664        MF->erase(SL->SwitchCases[I].ThisBB);665 666      SL->SwitchCases.clear();667    }668  }669 670  // Create a CaseBlock record representing this branch.671  SwitchCG::CaseBlock CB(CmpInst::ICMP_EQ, false, CondVal,672                         ConstantInt::getTrue(MF->getFunction().getContext()),673                         nullptr, Succ0MBB, Succ1MBB, &CurMBB,674                         CurBuilder->getDebugLoc());675 676  // Use emitSwitchCase to actually insert the fast branch sequence for this677  // cond branch.678  emitSwitchCase(CB, &CurMBB, *CurBuilder);679  return true;680}681 682void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src,683                                        MachineBasicBlock *Dst,684                                        BranchProbability Prob) {685  if (!FuncInfo.BPI) {686    Src->addSuccessorWithoutProb(Dst);687    return;688  }689  if (Prob.isUnknown())690    Prob = getEdgeProbability(Src, Dst);691  Src->addSuccessor(Dst, Prob);692}693 694BranchProbability695IRTranslator::getEdgeProbability(const MachineBasicBlock *Src,696                                 const MachineBasicBlock *Dst) const {697  const BasicBlock *SrcBB = Src->getBasicBlock();698  const BasicBlock *DstBB = Dst->getBasicBlock();699  if (!FuncInfo.BPI) {700    // If BPI is not available, set the default probability as 1 / N, where N is701    // the number of successors.702    auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);703    return BranchProbability(1, SuccSize);704  }705  return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB);706}707 708bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) {709  using namespace SwitchCG;710  // Extract cases from the switch.711  const SwitchInst &SI = cast<SwitchInst>(U);712  BranchProbabilityInfo *BPI = FuncInfo.BPI;713  CaseClusterVector Clusters;714  Clusters.reserve(SI.getNumCases());715  for (const auto &I : SI.cases()) {716    MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor());717    assert(Succ && "Could not find successor mbb in mapping");718    const ConstantInt *CaseVal = I.getCaseValue();719    BranchProbability Prob =720        BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())721            : BranchProbability(1, SI.getNumCases() + 1);722    Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));723  }724 725  MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest());726 727  // Cluster adjacent cases with the same destination. We do this at all728  // optimization levels because it's cheap to do and will make codegen faster729  // if there are many clusters.730  sortAndRangeify(Clusters);731 732  MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent());733 734  // If there is only the default destination, jump there directly.735  if (Clusters.empty()) {736    SwitchMBB->addSuccessor(DefaultMBB);737    if (DefaultMBB != SwitchMBB->getNextNode())738      MIB.buildBr(*DefaultMBB);739    return true;740  }741 742  SL->findJumpTables(Clusters, &SI, std::nullopt, DefaultMBB, nullptr, nullptr);743  SL->findBitTestClusters(Clusters, &SI);744 745  LLVM_DEBUG({746    dbgs() << "Case clusters: ";747    for (const CaseCluster &C : Clusters) {748      if (C.Kind == CC_JumpTable)749        dbgs() << "JT:";750      if (C.Kind == CC_BitTests)751        dbgs() << "BT:";752 753      C.Low->getValue().print(dbgs(), true);754      if (C.Low != C.High) {755        dbgs() << '-';756        C.High->getValue().print(dbgs(), true);757      }758      dbgs() << ' ';759    }760    dbgs() << '\n';761  });762 763  assert(!Clusters.empty());764  SwitchWorkList WorkList;765  CaseClusterIt First = Clusters.begin();766  CaseClusterIt Last = Clusters.end() - 1;767  auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB);768  WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb});769 770  while (!WorkList.empty()) {771    SwitchWorkListItem W = WorkList.pop_back_val();772 773    unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;774    // For optimized builds, lower large range as a balanced binary tree.775    if (NumClusters > 3 &&776        MF->getTarget().getOptLevel() != CodeGenOptLevel::None &&777        !DefaultMBB->getParent()->getFunction().hasMinSize()) {778      splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB, MIB);779      continue;780    }781 782    if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB))783      return false;784  }785  return true;786}787 788void IRTranslator::splitWorkItem(SwitchCG::SwitchWorkList &WorkList,789                                 const SwitchCG::SwitchWorkListItem &W,790                                 Value *Cond, MachineBasicBlock *SwitchMBB,791                                 MachineIRBuilder &MIB) {792  using namespace SwitchCG;793  assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&794         "Clusters not sorted?");795  assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!");796 797  auto [LastLeft, FirstRight, LeftProb, RightProb] =798      SL->computeSplitWorkItemInfo(W);799 800  // Use the first element on the right as pivot since we will make less-than801  // comparisons against it.802  CaseClusterIt PivotCluster = FirstRight;803  assert(PivotCluster > W.FirstCluster);804  assert(PivotCluster <= W.LastCluster);805 806  CaseClusterIt FirstLeft = W.FirstCluster;807  CaseClusterIt LastRight = W.LastCluster;808 809  const ConstantInt *Pivot = PivotCluster->Low;810 811  // New blocks will be inserted immediately after the current one.812  MachineFunction::iterator BBI(W.MBB);813  ++BBI;814 815  // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,816  // we can branch to its destination directly if it's squeezed exactly in817  // between the known lower bound and Pivot - 1.818  MachineBasicBlock *LeftMBB;819  if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&820      FirstLeft->Low == W.GE &&821      (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {822    LeftMBB = FirstLeft->MBB;823  } else {824    LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());825    FuncInfo.MF->insert(BBI, LeftMBB);826    WorkList.push_back(827        {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});828  }829 830  // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a831  // single cluster, RHS.Low == Pivot, and we can branch to its destination832  // directly if RHS.High equals the current upper bound.833  MachineBasicBlock *RightMBB;834  if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT &&835      (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {836    RightMBB = FirstRight->MBB;837  } else {838    RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());839    FuncInfo.MF->insert(BBI, RightMBB);840    WorkList.push_back(841        {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});842  }843 844  // Create the CaseBlock record that will be used to lower the branch.845  CaseBlock CB(ICmpInst::Predicate::ICMP_SLT, false, Cond, Pivot, nullptr,846               LeftMBB, RightMBB, W.MBB, MIB.getDebugLoc(), LeftProb,847               RightProb);848 849  if (W.MBB == SwitchMBB)850    emitSwitchCase(CB, SwitchMBB, MIB);851  else852    SL->SwitchCases.push_back(CB);853}854 855void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT,856                                 MachineBasicBlock *MBB) {857  // Emit the code for the jump table858  assert(JT.Reg && "Should lower JT Header first!");859  MachineIRBuilder MIB(*MBB->getParent());860  MIB.setMBB(*MBB);861  MIB.setDebugLoc(CurBuilder->getDebugLoc());862 863  Type *PtrIRTy = PointerType::getUnqual(MF->getFunction().getContext());864  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);865 866  auto Table = MIB.buildJumpTable(PtrTy, JT.JTI);867  MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg);868}869 870bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT,871                                       SwitchCG::JumpTableHeader &JTH,872                                       MachineBasicBlock *HeaderBB) {873  MachineIRBuilder MIB(*HeaderBB->getParent());874  MIB.setMBB(*HeaderBB);875  MIB.setDebugLoc(CurBuilder->getDebugLoc());876 877  const Value &SValue = *JTH.SValue;878  // Subtract the lowest switch case value from the value being switched on.879  const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL);880  Register SwitchOpReg = getOrCreateVReg(SValue);881  auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First);882  auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst);883 884  // This value may be smaller or larger than the target's pointer type, and885  // therefore require extension or truncating.886  auto *PtrIRTy = PointerType::getUnqual(SValue.getContext());887  const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy));888  Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub);889 890  JT.Reg = Sub.getReg(0);891 892  if (JTH.FallthroughUnreachable) {893    if (JT.MBB != HeaderBB->getNextNode())894      MIB.buildBr(*JT.MBB);895    return true;896  }897 898  // Emit the range check for the jump table, and branch to the default block899  // for the switch statement if the value being switched on exceeds the900  // largest case in the switch.901  auto Cst = getOrCreateVReg(902      *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First));903  Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0);904  auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst);905 906  auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default);907 908  // Avoid emitting unnecessary branches to the next block.909  if (JT.MBB != HeaderBB->getNextNode())910    BrCond = MIB.buildBr(*JT.MBB);911  return true;912}913 914void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,915                                  MachineBasicBlock *SwitchBB,916                                  MachineIRBuilder &MIB) {917  Register CondLHS = getOrCreateVReg(*CB.CmpLHS);918  Register Cond;919  DebugLoc OldDbgLoc = MIB.getDebugLoc();920  MIB.setDebugLoc(CB.DbgLoc);921  MIB.setMBB(*CB.ThisBB);922 923  if (CB.PredInfo.NoCmp) {924    // Branch or fall through to TrueBB.925    addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);926    addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},927                      CB.ThisBB);928    CB.ThisBB->normalizeSuccProbs();929    if (CB.TrueBB != CB.ThisBB->getNextNode())930      MIB.buildBr(*CB.TrueBB);931    MIB.setDebugLoc(OldDbgLoc);932    return;933  }934 935  const LLT i1Ty = LLT::scalar(1);936  // Build the compare.937  if (!CB.CmpMHS) {938    const auto *CI = dyn_cast<ConstantInt>(CB.CmpRHS);939    // For conditional branch lowering, we might try to do something silly like940    // emit an G_ICMP to compare an existing G_ICMP i1 result with true. If so,941    // just re-use the existing condition vreg.942    if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&943        CB.PredInfo.Pred == CmpInst::ICMP_EQ) {944      Cond = CondLHS;945    } else {946      Register CondRHS = getOrCreateVReg(*CB.CmpRHS);947      if (CmpInst::isFPPredicate(CB.PredInfo.Pred))948        Cond =949            MIB.buildFCmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);950      else951        Cond =952            MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);953    }954  } else {955    assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&956           "Can only handle SLE ranges");957 958    const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();959    const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();960 961    Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);962    if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {963      Register CondRHS = getOrCreateVReg(*CB.CmpRHS);964      Cond =965          MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);966    } else {967      const LLT CmpTy = MRI->getType(CmpOpReg);968      auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);969      auto Diff = MIB.buildConstant(CmpTy, High - Low);970      Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0);971    }972  }973 974  // Update successor info975  addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb);976 977  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()},978                    CB.ThisBB);979 980  // TrueBB and FalseBB are always different unless the incoming IR is981  // degenerate. This only happens when running llc on weird IR.982  if (CB.TrueBB != CB.FalseBB)983    addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb);984  CB.ThisBB->normalizeSuccProbs();985 986  addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},987                    CB.ThisBB);988 989  MIB.buildBrCond(Cond, *CB.TrueBB);990  MIB.buildBr(*CB.FalseBB);991  MIB.setDebugLoc(OldDbgLoc);992}993 994bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,995                                          MachineBasicBlock *SwitchMBB,996                                          MachineBasicBlock *CurMBB,997                                          MachineBasicBlock *DefaultMBB,998                                          MachineIRBuilder &MIB,999                                          MachineFunction::iterator BBI,1000                                          BranchProbability UnhandledProbs,1001                                          SwitchCG::CaseClusterIt I,1002                                          MachineBasicBlock *Fallthrough,1003                                          bool FallthroughUnreachable) {1004  using namespace SwitchCG;1005  MachineFunction *CurMF = SwitchMBB->getParent();1006  // FIXME: Optimize away range check based on pivot comparisons.1007  JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;1008  SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;1009  BranchProbability DefaultProb = W.DefaultProb;1010 1011  // The jump block hasn't been inserted yet; insert it here.1012  MachineBasicBlock *JumpMBB = JT->MBB;1013  CurMF->insert(BBI, JumpMBB);1014 1015  // Since the jump table block is separate from the switch block, we need1016  // to keep track of it as a machine predecessor to the default block,1017  // otherwise we lose the phi edges.1018  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},1019                    CurMBB);1020  addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()},1021                    JumpMBB);1022 1023  auto JumpProb = I->Prob;1024  auto FallthroughProb = UnhandledProbs;1025 1026  // If the default statement is a target of the jump table, we evenly1027  // distribute the default probability to successors of CurMBB. Also1028  // update the probability on the edge from JumpMBB to Fallthrough.1029  for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),1030                                        SE = JumpMBB->succ_end();1031       SI != SE; ++SI) {1032    if (*SI == DefaultMBB) {1033      JumpProb += DefaultProb / 2;1034      FallthroughProb -= DefaultProb / 2;1035      JumpMBB->setSuccProbability(SI, DefaultProb / 2);1036      JumpMBB->normalizeSuccProbs();1037    } else {1038      // Also record edges from the jump table block to it's successors.1039      addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()},1040                        JumpMBB);1041    }1042  }1043 1044  if (FallthroughUnreachable)1045    JTH->FallthroughUnreachable = true;1046 1047  if (!JTH->FallthroughUnreachable)1048    addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);1049  addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);1050  CurMBB->normalizeSuccProbs();1051 1052  // The jump table header will be inserted in our current block, do the1053  // range check, and fall through to our fallthrough block.1054  JTH->HeaderBB = CurMBB;1055  JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.1056 1057  // If we're in the right place, emit the jump table header right now.1058  if (CurMBB == SwitchMBB) {1059    if (!emitJumpTableHeader(*JT, *JTH, CurMBB))1060      return false;1061    JTH->Emitted = true;1062  }1063  return true;1064}1065bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,1066                                            Value *Cond,1067                                            MachineBasicBlock *Fallthrough,1068                                            bool FallthroughUnreachable,1069                                            BranchProbability UnhandledProbs,1070                                            MachineBasicBlock *CurMBB,1071                                            MachineIRBuilder &MIB,1072                                            MachineBasicBlock *SwitchMBB) {1073  using namespace SwitchCG;1074  const Value *RHS, *LHS, *MHS;1075  CmpInst::Predicate Pred;1076  if (I->Low == I->High) {1077    // Check Cond == I->Low.1078    Pred = CmpInst::ICMP_EQ;1079    LHS = Cond;1080    RHS = I->Low;1081    MHS = nullptr;1082  } else {1083    // Check I->Low <= Cond <= I->High.1084    Pred = CmpInst::ICMP_SLE;1085    LHS = I->Low;1086    MHS = Cond;1087    RHS = I->High;1088  }1089 1090  // If Fallthrough is unreachable, fold away the comparison.1091  // The false probability is the sum of all unhandled cases.1092  CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough,1093               CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs);1094 1095  emitSwitchCase(CB, SwitchMBB, MIB);1096  return true;1097}1098 1099void IRTranslator::emitBitTestHeader(SwitchCG::BitTestBlock &B,1100                                     MachineBasicBlock *SwitchBB) {1101  MachineIRBuilder &MIB = *CurBuilder;1102  MIB.setMBB(*SwitchBB);1103 1104  // Subtract the minimum value.1105  Register SwitchOpReg = getOrCreateVReg(*B.SValue);1106 1107  LLT SwitchOpTy = MRI->getType(SwitchOpReg);1108  Register MinValReg = MIB.buildConstant(SwitchOpTy, B.First).getReg(0);1109  auto RangeSub = MIB.buildSub(SwitchOpTy, SwitchOpReg, MinValReg);1110 1111  Type *PtrIRTy = PointerType::getUnqual(MF->getFunction().getContext());1112  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);1113 1114  LLT MaskTy = SwitchOpTy;1115  if (MaskTy.getSizeInBits() > PtrTy.getSizeInBits() ||1116      !llvm::has_single_bit<uint32_t>(MaskTy.getSizeInBits()))1117    MaskTy = LLT::scalar(PtrTy.getSizeInBits());1118  else {1119    // Ensure that the type will fit the mask value.1120    for (const SwitchCG::BitTestCase &Case : B.Cases) {1121      if (!isUIntN(SwitchOpTy.getSizeInBits(), Case.Mask)) {1122        // Switch table case range are encoded into series of masks.1123        // Just use pointer type, it's guaranteed to fit.1124        MaskTy = LLT::scalar(PtrTy.getSizeInBits());1125        break;1126      }1127    }1128  }1129  Register SubReg = RangeSub.getReg(0);1130  if (SwitchOpTy != MaskTy)1131    SubReg = MIB.buildZExtOrTrunc(MaskTy, SubReg).getReg(0);1132 1133  B.RegVT = getMVTForLLT(MaskTy);1134  B.Reg = SubReg;1135 1136  MachineBasicBlock *MBB = B.Cases[0].ThisBB;1137 1138  if (!B.FallthroughUnreachable)1139    addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);1140  addSuccessorWithProb(SwitchBB, MBB, B.Prob);1141 1142  SwitchBB->normalizeSuccProbs();1143 1144  if (!B.FallthroughUnreachable) {1145    // Conditional branch to the default block.1146    auto RangeCst = MIB.buildConstant(SwitchOpTy, B.Range);1147    auto RangeCmp = MIB.buildICmp(CmpInst::Predicate::ICMP_UGT, LLT::scalar(1),1148                                  RangeSub, RangeCst);1149    MIB.buildBrCond(RangeCmp, *B.Default);1150  }1151 1152  // Avoid emitting unnecessary branches to the next block.1153  if (MBB != SwitchBB->getNextNode())1154    MIB.buildBr(*MBB);1155}1156 1157void IRTranslator::emitBitTestCase(SwitchCG::BitTestBlock &BB,1158                                   MachineBasicBlock *NextMBB,1159                                   BranchProbability BranchProbToNext,1160                                   Register Reg, SwitchCG::BitTestCase &B,1161                                   MachineBasicBlock *SwitchBB) {1162  MachineIRBuilder &MIB = *CurBuilder;1163  MIB.setMBB(*SwitchBB);1164 1165  LLT SwitchTy = getLLTForMVT(BB.RegVT);1166  Register Cmp;1167  unsigned PopCount = llvm::popcount(B.Mask);1168  if (PopCount == 1) {1169    // Testing for a single bit; just compare the shift count with what it1170    // would need to be to shift a 1 bit in that position.1171    auto MaskTrailingZeros =1172        MIB.buildConstant(SwitchTy, llvm::countr_zero(B.Mask));1173    Cmp =1174        MIB.buildICmp(ICmpInst::ICMP_EQ, LLT::scalar(1), Reg, MaskTrailingZeros)1175            .getReg(0);1176  } else if (PopCount == BB.Range) {1177    // There is only one zero bit in the range, test for it directly.1178    auto MaskTrailingOnes =1179        MIB.buildConstant(SwitchTy, llvm::countr_one(B.Mask));1180    Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Reg, MaskTrailingOnes)1181              .getReg(0);1182  } else {1183    // Make desired shift.1184    auto CstOne = MIB.buildConstant(SwitchTy, 1);1185    auto SwitchVal = MIB.buildShl(SwitchTy, CstOne, Reg);1186 1187    // Emit bit tests and jumps.1188    auto CstMask = MIB.buildConstant(SwitchTy, B.Mask);1189    auto AndOp = MIB.buildAnd(SwitchTy, SwitchVal, CstMask);1190    auto CstZero = MIB.buildConstant(SwitchTy, 0);1191    Cmp = MIB.buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), AndOp, CstZero)1192              .getReg(0);1193  }1194 1195  // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.1196  addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);1197  // The branch probability from SwitchBB to NextMBB is BranchProbToNext.1198  addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);1199  // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is1200  // one as they are relative probabilities (and thus work more like weights),1201  // and hence we need to normalize them to let the sum of them become one.1202  SwitchBB->normalizeSuccProbs();1203 1204  // Record the fact that the IR edge from the header to the bit test target1205  // will go through our new block. Neeeded for PHIs to have nodes added.1206  addMachineCFGPred({BB.Parent->getBasicBlock(), B.TargetBB->getBasicBlock()},1207                    SwitchBB);1208 1209  MIB.buildBrCond(Cmp, *B.TargetBB);1210 1211  // Avoid emitting unnecessary branches to the next block.1212  if (NextMBB != SwitchBB->getNextNode())1213    MIB.buildBr(*NextMBB);1214}1215 1216bool IRTranslator::lowerBitTestWorkItem(1217    SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,1218    MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,1219    MachineIRBuilder &MIB, MachineFunction::iterator BBI,1220    BranchProbability DefaultProb, BranchProbability UnhandledProbs,1221    SwitchCG::CaseClusterIt I, MachineBasicBlock *Fallthrough,1222    bool FallthroughUnreachable) {1223  using namespace SwitchCG;1224  MachineFunction *CurMF = SwitchMBB->getParent();1225  // FIXME: Optimize away range check based on pivot comparisons.1226  BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];1227  // The bit test blocks haven't been inserted yet; insert them here.1228  for (BitTestCase &BTC : BTB->Cases)1229    CurMF->insert(BBI, BTC.ThisBB);1230 1231  // Fill in fields of the BitTestBlock.1232  BTB->Parent = CurMBB;1233  BTB->Default = Fallthrough;1234 1235  BTB->DefaultProb = UnhandledProbs;1236  // If the cases in bit test don't form a contiguous range, we evenly1237  // distribute the probability on the edge to Fallthrough to two1238  // successors of CurMBB.1239  if (!BTB->ContiguousRange) {1240    BTB->Prob += DefaultProb / 2;1241    BTB->DefaultProb -= DefaultProb / 2;1242  }1243 1244  if (FallthroughUnreachable)1245    BTB->FallthroughUnreachable = true;1246 1247  // If we're in the right place, emit the bit test header right now.1248  if (CurMBB == SwitchMBB) {1249    emitBitTestHeader(*BTB, SwitchMBB);1250    BTB->Emitted = true;1251  }1252  return true;1253}1254 1255bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W,1256                                       Value *Cond,1257                                       MachineBasicBlock *SwitchMBB,1258                                       MachineBasicBlock *DefaultMBB,1259                                       MachineIRBuilder &MIB) {1260  using namespace SwitchCG;1261  MachineFunction *CurMF = FuncInfo.MF;1262  MachineBasicBlock *NextMBB = nullptr;1263  MachineFunction::iterator BBI(W.MBB);1264  if (++BBI != FuncInfo.MF->end())1265    NextMBB = &*BBI;1266 1267  if (EnableOpts) {1268    // Here, we order cases by probability so the most likely case will be1269    // checked first. However, two clusters can have the same probability in1270    // which case their relative ordering is non-deterministic. So we use Low1271    // as a tie-breaker as clusters are guaranteed to never overlap.1272    llvm::sort(W.FirstCluster, W.LastCluster + 1,1273               [](const CaseCluster &a, const CaseCluster &b) {1274                 return a.Prob != b.Prob1275                            ? a.Prob > b.Prob1276                            : a.Low->getValue().slt(b.Low->getValue());1277               });1278 1279    // Rearrange the case blocks so that the last one falls through if possible1280    // without changing the order of probabilities.1281    for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) {1282      --I;1283      if (I->Prob > W.LastCluster->Prob)1284        break;1285      if (I->Kind == CC_Range && I->MBB == NextMBB) {1286        std::swap(*I, *W.LastCluster);1287        break;1288      }1289    }1290  }1291 1292  // Compute total probability.1293  BranchProbability DefaultProb = W.DefaultProb;1294  BranchProbability UnhandledProbs = DefaultProb;1295  for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)1296    UnhandledProbs += I->Prob;1297 1298  MachineBasicBlock *CurMBB = W.MBB;1299  for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {1300    bool FallthroughUnreachable = false;1301    MachineBasicBlock *Fallthrough;1302    if (I == W.LastCluster) {1303      // For the last cluster, fall through to the default destination.1304      Fallthrough = DefaultMBB;1305      FallthroughUnreachable = isa<UnreachableInst>(1306          DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg());1307    } else {1308      Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());1309      CurMF->insert(BBI, Fallthrough);1310    }1311    UnhandledProbs -= I->Prob;1312 1313    switch (I->Kind) {1314    case CC_BitTests: {1315      if (!lowerBitTestWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,1316                                DefaultProb, UnhandledProbs, I, Fallthrough,1317                                FallthroughUnreachable)) {1318        LLVM_DEBUG(dbgs() << "Failed to lower bit test for switch");1319        return false;1320      }1321      break;1322    }1323 1324    case CC_JumpTable: {1325      if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI,1326                                  UnhandledProbs, I, Fallthrough,1327                                  FallthroughUnreachable)) {1328        LLVM_DEBUG(dbgs() << "Failed to lower jump table");1329        return false;1330      }1331      break;1332    }1333    case CC_Range: {1334      if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough,1335                                    FallthroughUnreachable, UnhandledProbs,1336                                    CurMBB, MIB, SwitchMBB)) {1337        LLVM_DEBUG(dbgs() << "Failed to lower switch range");1338        return false;1339      }1340      break;1341    }1342    }1343    CurMBB = Fallthrough;1344  }1345 1346  return true;1347}1348 1349bool IRTranslator::translateIndirectBr(const User &U,1350                                       MachineIRBuilder &MIRBuilder) {1351  const IndirectBrInst &BrInst = cast<IndirectBrInst>(U);1352 1353  const Register Tgt = getOrCreateVReg(*BrInst.getAddress());1354  MIRBuilder.buildBrIndirect(Tgt);1355 1356  // Link successors.1357  SmallPtrSet<const BasicBlock *, 32> AddedSuccessors;1358  MachineBasicBlock &CurBB = MIRBuilder.getMBB();1359  for (const BasicBlock *Succ : successors(&BrInst)) {1360    // It's legal for indirectbr instructions to have duplicate blocks in the1361    // destination list. We don't allow this in MIR. Skip anything that's1362    // already a successor.1363    if (!AddedSuccessors.insert(Succ).second)1364      continue;1365    CurBB.addSuccessor(&getMBB(*Succ));1366  }1367 1368  return true;1369}1370 1371static bool isSwiftError(const Value *V) {1372  if (auto Arg = dyn_cast<Argument>(V))1373    return Arg->hasSwiftErrorAttr();1374  if (auto AI = dyn_cast<AllocaInst>(V))1375    return AI->isSwiftError();1376  return false;1377}1378 1379bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {1380  const LoadInst &LI = cast<LoadInst>(U);1381  TypeSize StoreSize = DL->getTypeStoreSize(LI.getType());1382  if (StoreSize.isZero())1383    return true;1384 1385  ArrayRef<Register> Regs = getOrCreateVRegs(LI);1386  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);1387  Register Base = getOrCreateVReg(*LI.getPointerOperand());1388  AAMDNodes AAInfo = LI.getAAMetadata();1389 1390  const Value *Ptr = LI.getPointerOperand();1391  Type *OffsetIRTy = DL->getIndexType(Ptr->getType());1392  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);1393 1394  if (CLI->supportSwiftError() && isSwiftError(Ptr)) {1395    assert(Regs.size() == 1 && "swifterror should be single pointer");1396    Register VReg =1397        SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), Ptr);1398    MIRBuilder.buildCopy(Regs[0], VReg);1399    return true;1400  }1401 1402  MachineMemOperand::Flags Flags =1403      TLI->getLoadMemOperandFlags(LI, *DL, AC, LibInfo);1404  if (AA && !(Flags & MachineMemOperand::MOInvariant)) {1405    if (AA->pointsToConstantMemory(1406            MemoryLocation(Ptr, LocationSize::precise(StoreSize), AAInfo))) {1407      Flags |= MachineMemOperand::MOInvariant;1408    }1409  }1410 1411  const MDNode *Ranges =1412      Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;1413  for (unsigned i = 0; i < Regs.size(); ++i) {1414    Register Addr;1415    MIRBuilder.materializeObjectPtrOffset(Addr, Base, OffsetTy, Offsets[i]);1416 1417    MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);1418    Align BaseAlign = getMemOpAlign(LI);1419    auto MMO =1420        MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),1421                                 commonAlignment(BaseAlign, Offsets[i]), AAInfo,1422                                 Ranges, LI.getSyncScopeID(), LI.getOrdering());1423    MIRBuilder.buildLoad(Regs[i], Addr, *MMO);1424  }1425 1426  return true;1427}1428 1429bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {1430  const StoreInst &SI = cast<StoreInst>(U);1431  if (DL->getTypeStoreSize(SI.getValueOperand()->getType()).isZero())1432    return true;1433 1434  ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());1435  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());1436  Register Base = getOrCreateVReg(*SI.getPointerOperand());1437 1438  Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType());1439  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);1440 1441  if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {1442    assert(Vals.size() == 1 && "swifterror should be single pointer");1443 1444    Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(),1445                                                    SI.getPointerOperand());1446    MIRBuilder.buildCopy(VReg, Vals[0]);1447    return true;1448  }1449 1450  MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, *DL);1451 1452  for (unsigned i = 0; i < Vals.size(); ++i) {1453    Register Addr;1454    MIRBuilder.materializeObjectPtrOffset(Addr, Base, OffsetTy, Offsets[i]);1455 1456    MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i]);1457    Align BaseAlign = getMemOpAlign(SI);1458    auto MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Vals[i]),1459                                        commonAlignment(BaseAlign, Offsets[i]),1460                                        SI.getAAMetadata(), nullptr,1461                                        SI.getSyncScopeID(), SI.getOrdering());1462    MIRBuilder.buildStore(Vals[i], Addr, *MMO);1463  }1464  return true;1465}1466 1467static uint64_t getOffsetFromIndices(const User &U, const DataLayout &DL) {1468  const Value *Src = U.getOperand(0);1469  Type *Int32Ty = Type::getInt32Ty(U.getContext());1470 1471  // getIndexedOffsetInType is designed for GEPs, so the first index is the1472  // usual array element rather than looking into the actual aggregate.1473  SmallVector<Value *, 1> Indices;1474  Indices.push_back(ConstantInt::get(Int32Ty, 0));1475 1476  if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(&U)) {1477    for (auto Idx : EVI->indices())1478      Indices.push_back(ConstantInt::get(Int32Ty, Idx));1479  } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(&U)) {1480    for (auto Idx : IVI->indices())1481      Indices.push_back(ConstantInt::get(Int32Ty, Idx));1482  } else {1483    llvm::append_range(Indices, drop_begin(U.operands()));1484  }1485 1486  return static_cast<uint64_t>(1487      DL.getIndexedOffsetInType(Src->getType(), Indices));1488}1489 1490bool IRTranslator::translateExtractValue(const User &U,1491                                         MachineIRBuilder &MIRBuilder) {1492  const Value *Src = U.getOperand(0);1493  uint64_t Offset = getOffsetFromIndices(U, *DL);1494  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);1495  ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*Src);1496  unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin();1497  auto &DstRegs = allocateVRegs(U);1498 1499  for (unsigned i = 0; i < DstRegs.size(); ++i)1500    DstRegs[i] = SrcRegs[Idx++];1501 1502  return true;1503}1504 1505bool IRTranslator::translateInsertValue(const User &U,1506                                        MachineIRBuilder &MIRBuilder) {1507  const Value *Src = U.getOperand(0);1508  uint64_t Offset = getOffsetFromIndices(U, *DL);1509  auto &DstRegs = allocateVRegs(U);1510  ArrayRef<uint64_t> DstOffsets = *VMap.getOffsets(U);1511  ArrayRef<Register> SrcRegs = getOrCreateVRegs(*Src);1512  ArrayRef<Register> InsertedRegs = getOrCreateVRegs(*U.getOperand(1));1513  auto *InsertedIt = InsertedRegs.begin();1514 1515  for (unsigned i = 0; i < DstRegs.size(); ++i) {1516    if (DstOffsets[i] >= Offset && InsertedIt != InsertedRegs.end())1517      DstRegs[i] = *InsertedIt++;1518    else1519      DstRegs[i] = SrcRegs[i];1520  }1521 1522  return true;1523}1524 1525bool IRTranslator::translateSelect(const User &U,1526                                   MachineIRBuilder &MIRBuilder) {1527  Register Tst = getOrCreateVReg(*U.getOperand(0));1528  ArrayRef<Register> ResRegs = getOrCreateVRegs(U);1529  ArrayRef<Register> Op0Regs = getOrCreateVRegs(*U.getOperand(1));1530  ArrayRef<Register> Op1Regs = getOrCreateVRegs(*U.getOperand(2));1531 1532  uint32_t Flags = 0;1533  if (const SelectInst *SI = dyn_cast<SelectInst>(&U))1534    Flags = MachineInstr::copyFlagsFromInstruction(*SI);1535 1536  for (unsigned i = 0; i < ResRegs.size(); ++i) {1537    MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i], Flags);1538  }1539 1540  return true;1541}1542 1543bool IRTranslator::translateCopy(const User &U, const Value &V,1544                                 MachineIRBuilder &MIRBuilder) {1545  Register Src = getOrCreateVReg(V);1546  auto &Regs = *VMap.getVRegs(U);1547  if (Regs.empty()) {1548    Regs.push_back(Src);1549    VMap.getOffsets(U)->push_back(0);1550  } else {1551    // If we already assigned a vreg for this instruction, we can't change that.1552    // Emit a copy to satisfy the users we already emitted.1553    MIRBuilder.buildCopy(Regs[0], Src);1554  }1555  return true;1556}1557 1558bool IRTranslator::translateBitCast(const User &U,1559                                    MachineIRBuilder &MIRBuilder) {1560  // If we're bitcasting to the source type, we can reuse the source vreg.1561  if (getLLTForType(*U.getOperand(0)->getType(), *DL) ==1562      getLLTForType(*U.getType(), *DL)) {1563    // If the source is a ConstantInt then it was probably created by1564    // ConstantHoisting and we should leave it alone.1565    if (isa<ConstantInt>(U.getOperand(0)))1566      return translateCast(TargetOpcode::G_CONSTANT_FOLD_BARRIER, U,1567                           MIRBuilder);1568    return translateCopy(U, *U.getOperand(0), MIRBuilder);1569  }1570 1571  return translateCast(TargetOpcode::G_BITCAST, U, MIRBuilder);1572}1573 1574bool IRTranslator::translateCast(unsigned Opcode, const User &U,1575                                 MachineIRBuilder &MIRBuilder) {1576  if (containsBF16Type(U) && !targetSupportsBF16Type(MF))1577    return false;1578 1579  uint32_t Flags = 0;1580  if (const Instruction *I = dyn_cast<Instruction>(&U))1581    Flags = MachineInstr::copyFlagsFromInstruction(*I);1582 1583  Register Op = getOrCreateVReg(*U.getOperand(0));1584  Register Res = getOrCreateVReg(U);1585  MIRBuilder.buildInstr(Opcode, {Res}, {Op}, Flags);1586  return true;1587}1588 1589bool IRTranslator::translateGetElementPtr(const User &U,1590                                          MachineIRBuilder &MIRBuilder) {1591  Value &Op0 = *U.getOperand(0);1592  Register BaseReg = getOrCreateVReg(Op0);1593  Type *PtrIRTy = Op0.getType();1594  LLT PtrTy = getLLTForType(*PtrIRTy, *DL);1595  Type *OffsetIRTy = DL->getIndexType(PtrIRTy);1596  LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);1597 1598  uint32_t PtrAddFlags = 0;1599  // Each PtrAdd generated to implement the GEP inherits its nuw, nusw, inbounds1600  // flags.1601  if (const Instruction *I = dyn_cast<Instruction>(&U))1602    PtrAddFlags = MachineInstr::copyFlagsFromInstruction(*I);1603 1604  auto PtrAddFlagsWithConst = [&](int64_t Offset) {1605    // For nusw/inbounds GEP with an offset that is nonnegative when interpreted1606    // as signed, assume there is no unsigned overflow.1607    if (Offset >= 0 && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap))1608      return PtrAddFlags | MachineInstr::MIFlag::NoUWrap;1609    return PtrAddFlags;1610  };1611 1612  // Normalize Vector GEP - all scalar operands should be converted to the1613  // splat vector.1614  unsigned VectorWidth = 0;1615 1616  // True if we should use a splat vector; using VectorWidth alone is not1617  // sufficient.1618  bool WantSplatVector = false;1619  if (auto *VT = dyn_cast<VectorType>(U.getType())) {1620    VectorWidth = cast<FixedVectorType>(VT)->getNumElements();1621    // We don't produce 1 x N vectors; those are treated as scalars.1622    WantSplatVector = VectorWidth > 1;1623  }1624 1625  // We might need to splat the base pointer into a vector if the offsets1626  // are vectors.1627  if (WantSplatVector && !PtrTy.isVector()) {1628    BaseReg = MIRBuilder1629                  .buildSplatBuildVector(LLT::fixed_vector(VectorWidth, PtrTy),1630                                         BaseReg)1631                  .getReg(0);1632    PtrIRTy = FixedVectorType::get(PtrIRTy, VectorWidth);1633    PtrTy = getLLTForType(*PtrIRTy, *DL);1634    OffsetIRTy = DL->getIndexType(PtrIRTy);1635    OffsetTy = getLLTForType(*OffsetIRTy, *DL);1636  }1637 1638  int64_t Offset = 0;1639  for (gep_type_iterator GTI = gep_type_begin(&U), E = gep_type_end(&U);1640       GTI != E; ++GTI) {1641    const Value *Idx = GTI.getOperand();1642    if (StructType *StTy = GTI.getStructTypeOrNull()) {1643      unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();1644      Offset += DL->getStructLayout(StTy)->getElementOffset(Field);1645      continue;1646    } else {1647      uint64_t ElementSize = GTI.getSequentialElementStride(*DL);1648 1649      // If this is a scalar constant or a splat vector of constants,1650      // handle it quickly.1651      if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {1652        if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {1653          Offset += ElementSize * *Val;1654          continue;1655        }1656      }1657 1658      if (Offset != 0) {1659        auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset);1660        BaseReg = MIRBuilder1661                      .buildPtrAdd(PtrTy, BaseReg, OffsetMIB.getReg(0),1662                                   PtrAddFlagsWithConst(Offset))1663                      .getReg(0);1664        Offset = 0;1665      }1666 1667      Register IdxReg = getOrCreateVReg(*Idx);1668      LLT IdxTy = MRI->getType(IdxReg);1669      if (IdxTy != OffsetTy) {1670        if (!IdxTy.isVector() && WantSplatVector) {1671          IdxReg = MIRBuilder1672                       .buildSplatBuildVector(OffsetTy.changeElementType(IdxTy),1673                                              IdxReg)1674                       .getReg(0);1675        }1676 1677        IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);1678      }1679 1680      // N = N + Idx * ElementSize;1681      // Avoid doing it for ElementSize of 1.1682      Register GepOffsetReg;1683      if (ElementSize != 1) {1684        auto ElementSizeMIB = MIRBuilder.buildConstant(1685            getLLTForType(*OffsetIRTy, *DL), ElementSize);1686 1687        // The multiplication is NUW if the GEP is NUW and NSW if the GEP is1688        // NUSW.1689        uint32_t ScaleFlags = PtrAddFlags & MachineInstr::MIFlag::NoUWrap;1690        if (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap)1691          ScaleFlags |= MachineInstr::MIFlag::NoSWrap;1692 1693        GepOffsetReg =1694            MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB, ScaleFlags)1695                .getReg(0);1696      } else {1697        GepOffsetReg = IdxReg;1698      }1699 1700      BaseReg =1701          MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg, PtrAddFlags)1702              .getReg(0);1703    }1704  }1705 1706  if (Offset != 0) {1707    auto OffsetMIB =1708        MIRBuilder.buildConstant(OffsetTy, Offset);1709 1710    MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0),1711                           PtrAddFlagsWithConst(Offset));1712    return true;1713  }1714 1715  MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);1716  return true;1717}1718 1719bool IRTranslator::translateMemFunc(const CallInst &CI,1720                                    MachineIRBuilder &MIRBuilder,1721                                    unsigned Opcode) {1722  const Value *SrcPtr = CI.getArgOperand(1);1723  // If the source is undef, then just emit a nop.1724  if (isa<UndefValue>(SrcPtr))1725    return true;1726 1727  SmallVector<Register, 3> SrcRegs;1728 1729  unsigned MinPtrSize = UINT_MAX;1730  for (auto AI = CI.arg_begin(), AE = CI.arg_end(); std::next(AI) != AE; ++AI) {1731    Register SrcReg = getOrCreateVReg(**AI);1732    LLT SrcTy = MRI->getType(SrcReg);1733    if (SrcTy.isPointer())1734      MinPtrSize = std::min<unsigned>(SrcTy.getSizeInBits(), MinPtrSize);1735    SrcRegs.push_back(SrcReg);1736  }1737 1738  LLT SizeTy = LLT::scalar(MinPtrSize);1739 1740  // The size operand should be the minimum of the pointer sizes.1741  Register &SizeOpReg = SrcRegs[SrcRegs.size() - 1];1742  if (MRI->getType(SizeOpReg) != SizeTy)1743    SizeOpReg = MIRBuilder.buildZExtOrTrunc(SizeTy, SizeOpReg).getReg(0);1744 1745  auto ICall = MIRBuilder.buildInstr(Opcode);1746  for (Register SrcReg : SrcRegs)1747    ICall.addUse(SrcReg);1748 1749  Align DstAlign;1750  Align SrcAlign;1751  unsigned IsVol =1752      cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1))->getZExtValue();1753 1754  ConstantInt *CopySize = nullptr;1755 1756  if (auto *MCI = dyn_cast<MemCpyInst>(&CI)) {1757    DstAlign = MCI->getDestAlign().valueOrOne();1758    SrcAlign = MCI->getSourceAlign().valueOrOne();1759    CopySize = dyn_cast<ConstantInt>(MCI->getArgOperand(2));1760  } else if (auto *MMI = dyn_cast<MemMoveInst>(&CI)) {1761    DstAlign = MMI->getDestAlign().valueOrOne();1762    SrcAlign = MMI->getSourceAlign().valueOrOne();1763    CopySize = dyn_cast<ConstantInt>(MMI->getArgOperand(2));1764  } else {1765    auto *MSI = cast<MemSetInst>(&CI);1766    DstAlign = MSI->getDestAlign().valueOrOne();1767  }1768 1769  if (Opcode != TargetOpcode::G_MEMCPY_INLINE) {1770    // We need to propagate the tail call flag from the IR inst as an argument.1771    // Otherwise, we have to pessimize and assume later that we cannot tail call1772    // any memory intrinsics.1773    ICall.addImm(CI.isTailCall() ? 1 : 0);1774  }1775 1776  // Create mem operands to store the alignment and volatile info.1777  MachineMemOperand::Flags LoadFlags = MachineMemOperand::MOLoad;1778  MachineMemOperand::Flags StoreFlags = MachineMemOperand::MOStore;1779  if (IsVol) {1780    LoadFlags |= MachineMemOperand::MOVolatile;1781    StoreFlags |= MachineMemOperand::MOVolatile;1782  }1783 1784  AAMDNodes AAInfo = CI.getAAMetadata();1785  if (AA && CopySize &&1786      AA->pointsToConstantMemory(MemoryLocation(1787          SrcPtr, LocationSize::precise(CopySize->getZExtValue()), AAInfo))) {1788    LoadFlags |= MachineMemOperand::MOInvariant;1789 1790    // FIXME: pointsToConstantMemory probably does not imply dereferenceable,1791    // but the previous usage implied it did. Probably should check1792    // isDereferenceableAndAlignedPointer.1793    LoadFlags |= MachineMemOperand::MODereferenceable;1794  }1795 1796  ICall.addMemOperand(1797      MF->getMachineMemOperand(MachinePointerInfo(CI.getArgOperand(0)),1798                               StoreFlags, 1, DstAlign, AAInfo));1799  if (Opcode != TargetOpcode::G_MEMSET)1800    ICall.addMemOperand(MF->getMachineMemOperand(1801        MachinePointerInfo(SrcPtr), LoadFlags, 1, SrcAlign, AAInfo));1802 1803  return true;1804}1805 1806bool IRTranslator::translateTrap(const CallInst &CI,1807                                 MachineIRBuilder &MIRBuilder,1808                                 unsigned Opcode) {1809  StringRef TrapFuncName =1810      CI.getAttributes().getFnAttr("trap-func-name").getValueAsString();1811  if (TrapFuncName.empty()) {1812    if (Opcode == TargetOpcode::G_UBSANTRAP) {1813      uint64_t Code = cast<ConstantInt>(CI.getOperand(0))->getZExtValue();1814      MIRBuilder.buildInstr(Opcode, {}, ArrayRef<llvm::SrcOp>{Code});1815    } else {1816      MIRBuilder.buildInstr(Opcode);1817    }1818    return true;1819  }1820 1821  CallLowering::CallLoweringInfo Info;1822  if (Opcode == TargetOpcode::G_UBSANTRAP)1823    Info.OrigArgs.push_back({getOrCreateVRegs(*CI.getArgOperand(0)),1824                             CI.getArgOperand(0)->getType(), 0});1825 1826  Info.Callee = MachineOperand::CreateES(TrapFuncName.data());1827  Info.CB = &CI;1828  Info.OrigRet = {Register(), Type::getVoidTy(CI.getContext()), 0};1829  return CLI->lowerCall(MIRBuilder, Info);1830}1831 1832bool IRTranslator::translateVectorInterleave2Intrinsic(1833    const CallInst &CI, MachineIRBuilder &MIRBuilder) {1834  assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 &&1835         "This function can only be called on the interleave2 intrinsic!");1836  // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG).1837  Register Op0 = getOrCreateVReg(*CI.getOperand(0));1838  Register Op1 = getOrCreateVReg(*CI.getOperand(1));1839  Register Res = getOrCreateVReg(CI);1840 1841  LLT OpTy = MRI->getType(Op0);1842  MIRBuilder.buildShuffleVector(Res, Op0, Op1,1843                                createInterleaveMask(OpTy.getNumElements(), 2));1844 1845  return true;1846}1847 1848bool IRTranslator::translateVectorDeinterleave2Intrinsic(1849    const CallInst &CI, MachineIRBuilder &MIRBuilder) {1850  assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 &&1851         "This function can only be called on the deinterleave2 intrinsic!");1852  // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to1853  // SelectionDAG).1854  Register Op = getOrCreateVReg(*CI.getOperand(0));1855  auto Undef = MIRBuilder.buildUndef(MRI->getType(Op));1856  ArrayRef<Register> Res = getOrCreateVRegs(CI);1857 1858  LLT ResTy = MRI->getType(Res[0]);1859  MIRBuilder.buildShuffleVector(Res[0], Op, Undef,1860                                createStrideMask(0, 2, ResTy.getNumElements()));1861  MIRBuilder.buildShuffleVector(Res[1], Op, Undef,1862                                createStrideMask(1, 2, ResTy.getNumElements()));1863 1864  return true;1865}1866 1867void IRTranslator::getStackGuard(Register DstReg,1868                                 MachineIRBuilder &MIRBuilder) {1869  Value *Global = TLI->getSDagStackGuard(*MF->getFunction().getParent());1870  if (!Global) {1871    LLVMContext &Ctx = MIRBuilder.getContext();1872    Ctx.diagnose(DiagnosticInfoGeneric("unable to lower stackguard"));1873    MIRBuilder.buildUndef(DstReg);1874    return;1875  }1876 1877  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();1878  MRI->setRegClass(DstReg, TRI->getPointerRegClass());1879  auto MIB =1880      MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD, {DstReg}, {});1881 1882  unsigned AddrSpace = Global->getType()->getPointerAddressSpace();1883  LLT PtrTy = LLT::pointer(AddrSpace, DL->getPointerSizeInBits(AddrSpace));1884 1885  MachinePointerInfo MPInfo(Global);1886  auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |1887               MachineMemOperand::MODereferenceable;1888  MachineMemOperand *MemRef = MF->getMachineMemOperand(1889      MPInfo, Flags, PtrTy, DL->getPointerABIAlignment(AddrSpace));1890  MIB.setMemRefs({MemRef});1891}1892 1893bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,1894                                              MachineIRBuilder &MIRBuilder) {1895  ArrayRef<Register> ResRegs = getOrCreateVRegs(CI);1896  MIRBuilder.buildInstr(1897      Op, {ResRegs[0], ResRegs[1]},1898      {getOrCreateVReg(*CI.getOperand(0)), getOrCreateVReg(*CI.getOperand(1))});1899 1900  return true;1901}1902 1903bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,1904                                                MachineIRBuilder &MIRBuilder) {1905  Register Dst = getOrCreateVReg(CI);1906  Register Src0 = getOrCreateVReg(*CI.getOperand(0));1907  Register Src1 = getOrCreateVReg(*CI.getOperand(1));1908  uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();1909  MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });1910  return true;1911}1912 1913unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {1914  switch (ID) {1915    default:1916      break;1917    case Intrinsic::acos:1918      return TargetOpcode::G_FACOS;1919    case Intrinsic::asin:1920      return TargetOpcode::G_FASIN;1921    case Intrinsic::atan:1922      return TargetOpcode::G_FATAN;1923    case Intrinsic::atan2:1924      return TargetOpcode::G_FATAN2;1925    case Intrinsic::bswap:1926      return TargetOpcode::G_BSWAP;1927    case Intrinsic::bitreverse:1928      return TargetOpcode::G_BITREVERSE;1929    case Intrinsic::fshl:1930      return TargetOpcode::G_FSHL;1931    case Intrinsic::fshr:1932      return TargetOpcode::G_FSHR;1933    case Intrinsic::ceil:1934      return TargetOpcode::G_FCEIL;1935    case Intrinsic::cos:1936      return TargetOpcode::G_FCOS;1937    case Intrinsic::cosh:1938      return TargetOpcode::G_FCOSH;1939    case Intrinsic::ctpop:1940      return TargetOpcode::G_CTPOP;1941    case Intrinsic::exp:1942      return TargetOpcode::G_FEXP;1943    case Intrinsic::exp2:1944      return TargetOpcode::G_FEXP2;1945    case Intrinsic::exp10:1946      return TargetOpcode::G_FEXP10;1947    case Intrinsic::fabs:1948      return TargetOpcode::G_FABS;1949    case Intrinsic::copysign:1950      return TargetOpcode::G_FCOPYSIGN;1951    case Intrinsic::minnum:1952      return TargetOpcode::G_FMINNUM;1953    case Intrinsic::maxnum:1954      return TargetOpcode::G_FMAXNUM;1955    case Intrinsic::minimum:1956      return TargetOpcode::G_FMINIMUM;1957    case Intrinsic::maximum:1958      return TargetOpcode::G_FMAXIMUM;1959    case Intrinsic::minimumnum:1960      return TargetOpcode::G_FMINIMUMNUM;1961    case Intrinsic::maximumnum:1962      return TargetOpcode::G_FMAXIMUMNUM;1963    case Intrinsic::canonicalize:1964      return TargetOpcode::G_FCANONICALIZE;1965    case Intrinsic::floor:1966      return TargetOpcode::G_FFLOOR;1967    case Intrinsic::fma:1968      return TargetOpcode::G_FMA;1969    case Intrinsic::log:1970      return TargetOpcode::G_FLOG;1971    case Intrinsic::log2:1972      return TargetOpcode::G_FLOG2;1973    case Intrinsic::log10:1974      return TargetOpcode::G_FLOG10;1975    case Intrinsic::ldexp:1976      return TargetOpcode::G_FLDEXP;1977    case Intrinsic::nearbyint:1978      return TargetOpcode::G_FNEARBYINT;1979    case Intrinsic::pow:1980      return TargetOpcode::G_FPOW;1981    case Intrinsic::powi:1982      return TargetOpcode::G_FPOWI;1983    case Intrinsic::rint:1984      return TargetOpcode::G_FRINT;1985    case Intrinsic::round:1986      return TargetOpcode::G_INTRINSIC_ROUND;1987    case Intrinsic::roundeven:1988      return TargetOpcode::G_INTRINSIC_ROUNDEVEN;1989    case Intrinsic::sin:1990      return TargetOpcode::G_FSIN;1991    case Intrinsic::sinh:1992      return TargetOpcode::G_FSINH;1993    case Intrinsic::sqrt:1994      return TargetOpcode::G_FSQRT;1995    case Intrinsic::tan:1996      return TargetOpcode::G_FTAN;1997    case Intrinsic::tanh:1998      return TargetOpcode::G_FTANH;1999    case Intrinsic::trunc:2000      return TargetOpcode::G_INTRINSIC_TRUNC;2001    case Intrinsic::readcyclecounter:2002      return TargetOpcode::G_READCYCLECOUNTER;2003    case Intrinsic::readsteadycounter:2004      return TargetOpcode::G_READSTEADYCOUNTER;2005    case Intrinsic::ptrmask:2006      return TargetOpcode::G_PTRMASK;2007    case Intrinsic::lrint:2008      return TargetOpcode::G_INTRINSIC_LRINT;2009    case Intrinsic::llrint:2010      return TargetOpcode::G_INTRINSIC_LLRINT;2011    // FADD/FMUL require checking the FMF, so are handled elsewhere.2012    case Intrinsic::vector_reduce_fmin:2013      return TargetOpcode::G_VECREDUCE_FMIN;2014    case Intrinsic::vector_reduce_fmax:2015      return TargetOpcode::G_VECREDUCE_FMAX;2016    case Intrinsic::vector_reduce_fminimum:2017      return TargetOpcode::G_VECREDUCE_FMINIMUM;2018    case Intrinsic::vector_reduce_fmaximum:2019      return TargetOpcode::G_VECREDUCE_FMAXIMUM;2020    case Intrinsic::vector_reduce_add:2021      return TargetOpcode::G_VECREDUCE_ADD;2022    case Intrinsic::vector_reduce_mul:2023      return TargetOpcode::G_VECREDUCE_MUL;2024    case Intrinsic::vector_reduce_and:2025      return TargetOpcode::G_VECREDUCE_AND;2026    case Intrinsic::vector_reduce_or:2027      return TargetOpcode::G_VECREDUCE_OR;2028    case Intrinsic::vector_reduce_xor:2029      return TargetOpcode::G_VECREDUCE_XOR;2030    case Intrinsic::vector_reduce_smax:2031      return TargetOpcode::G_VECREDUCE_SMAX;2032    case Intrinsic::vector_reduce_smin:2033      return TargetOpcode::G_VECREDUCE_SMIN;2034    case Intrinsic::vector_reduce_umax:2035      return TargetOpcode::G_VECREDUCE_UMAX;2036    case Intrinsic::vector_reduce_umin:2037      return TargetOpcode::G_VECREDUCE_UMIN;2038    case Intrinsic::experimental_vector_compress:2039      return TargetOpcode::G_VECTOR_COMPRESS;2040    case Intrinsic::lround:2041      return TargetOpcode::G_LROUND;2042    case Intrinsic::llround:2043      return TargetOpcode::G_LLROUND;2044    case Intrinsic::get_fpenv:2045      return TargetOpcode::G_GET_FPENV;2046    case Intrinsic::get_fpmode:2047      return TargetOpcode::G_GET_FPMODE;2048  }2049  return Intrinsic::not_intrinsic;2050}2051 2052bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,2053                                            Intrinsic::ID ID,2054                                            MachineIRBuilder &MIRBuilder) {2055 2056  unsigned Op = getSimpleIntrinsicOpcode(ID);2057 2058  // Is this a simple intrinsic?2059  if (Op == Intrinsic::not_intrinsic)2060    return false;2061 2062  // Yes. Let's translate it.2063  SmallVector<llvm::SrcOp, 4> VRegs;2064  for (const auto &Arg : CI.args())2065    VRegs.push_back(getOrCreateVReg(*Arg));2066 2067  MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,2068                        MachineInstr::copyFlagsFromInstruction(CI));2069  return true;2070}2071 2072// TODO: Include ConstainedOps.def when all strict instructions are defined.2073static unsigned getConstrainedOpcode(Intrinsic::ID ID) {2074  switch (ID) {2075  case Intrinsic::experimental_constrained_fadd:2076    return TargetOpcode::G_STRICT_FADD;2077  case Intrinsic::experimental_constrained_fsub:2078    return TargetOpcode::G_STRICT_FSUB;2079  case Intrinsic::experimental_constrained_fmul:2080    return TargetOpcode::G_STRICT_FMUL;2081  case Intrinsic::experimental_constrained_fdiv:2082    return TargetOpcode::G_STRICT_FDIV;2083  case Intrinsic::experimental_constrained_frem:2084    return TargetOpcode::G_STRICT_FREM;2085  case Intrinsic::experimental_constrained_fma:2086    return TargetOpcode::G_STRICT_FMA;2087  case Intrinsic::experimental_constrained_sqrt:2088    return TargetOpcode::G_STRICT_FSQRT;2089  case Intrinsic::experimental_constrained_ldexp:2090    return TargetOpcode::G_STRICT_FLDEXP;2091  default:2092    return 0;2093  }2094}2095 2096bool IRTranslator::translateConstrainedFPIntrinsic(2097  const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {2098  fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();2099 2100  unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());2101  if (!Opcode)2102    return false;2103 2104  uint32_t Flags = MachineInstr::copyFlagsFromInstruction(FPI);2105  if (EB == fp::ExceptionBehavior::ebIgnore)2106    Flags |= MachineInstr::NoFPExcept;2107 2108  SmallVector<llvm::SrcOp, 4> VRegs;2109  for (unsigned I = 0, E = FPI.getNonMetadataArgCount(); I != E; ++I)2110    VRegs.push_back(getOrCreateVReg(*FPI.getArgOperand(I)));2111 2112  MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(FPI)}, VRegs, Flags);2113  return true;2114}2115 2116std::optional<MCRegister> IRTranslator::getArgPhysReg(Argument &Arg) {2117  auto VRegs = getOrCreateVRegs(Arg);2118  if (VRegs.size() != 1)2119    return std::nullopt;2120 2121  // Arguments are lowered as a copy of a livein physical register.2122  auto *VRegDef = MF->getRegInfo().getVRegDef(VRegs[0]);2123  if (!VRegDef || !VRegDef->isCopy())2124    return std::nullopt;2125  return VRegDef->getOperand(1).getReg().asMCReg();2126}2127 2128bool IRTranslator::translateIfEntryValueArgument(bool isDeclare, Value *Val,2129                                                 const DILocalVariable *Var,2130                                                 const DIExpression *Expr,2131                                                 const DebugLoc &DL,2132                                                 MachineIRBuilder &MIRBuilder) {2133  auto *Arg = dyn_cast<Argument>(Val);2134  if (!Arg)2135    return false;2136 2137  if (!Expr->isEntryValue())2138    return false;2139 2140  std::optional<MCRegister> PhysReg = getArgPhysReg(*Arg);2141  if (!PhysReg) {2142    LLVM_DEBUG(dbgs() << "Dropping dbg." << (isDeclare ? "declare" : "value")2143                      << ": expression is entry_value but "2144                      << "couldn't find a physical register\n");2145    LLVM_DEBUG(dbgs() << *Var << "\n");2146    return true;2147  }2148 2149  if (isDeclare) {2150    // Append an op deref to account for the fact that this is a dbg_declare.2151    Expr = DIExpression::append(Expr, dwarf::DW_OP_deref);2152    MF->setVariableDbgInfo(Var, Expr, *PhysReg, DL);2153  } else {2154    MIRBuilder.buildDirectDbgValue(*PhysReg, Var, Expr);2155  }2156 2157  return true;2158}2159 2160static unsigned getConvOpcode(Intrinsic::ID ID) {2161  switch (ID) {2162  default:2163    llvm_unreachable("Unexpected intrinsic");2164  case Intrinsic::experimental_convergence_anchor:2165    return TargetOpcode::CONVERGENCECTRL_ANCHOR;2166  case Intrinsic::experimental_convergence_entry:2167    return TargetOpcode::CONVERGENCECTRL_ENTRY;2168  case Intrinsic::experimental_convergence_loop:2169    return TargetOpcode::CONVERGENCECTRL_LOOP;2170  }2171}2172 2173bool IRTranslator::translateConvergenceControlIntrinsic(2174    const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) {2175  MachineInstrBuilder MIB = MIRBuilder.buildInstr(getConvOpcode(ID));2176  Register OutputReg = getOrCreateConvergenceTokenVReg(CI);2177  MIB.addDef(OutputReg);2178 2179  if (ID == Intrinsic::experimental_convergence_loop) {2180    auto Bundle = CI.getOperandBundle(LLVMContext::OB_convergencectrl);2181    assert(Bundle && "Expected a convergence control token.");2182    Register InputReg =2183        getOrCreateConvergenceTokenVReg(*Bundle->Inputs[0].get());2184    MIB.addUse(InputReg);2185  }2186 2187  return true;2188}2189 2190bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,2191                                           MachineIRBuilder &MIRBuilder) {2192  if (auto *MI = dyn_cast<AnyMemIntrinsic>(&CI)) {2193    if (ORE->enabled()) {2194      if (MemoryOpRemark::canHandle(MI, *LibInfo)) {2195        MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);2196        R.visit(MI);2197      }2198    }2199  }2200 2201  // If this is a simple intrinsic (that is, we just need to add a def of2202  // a vreg, and uses for each arg operand, then translate it.2203  if (translateSimpleIntrinsic(CI, ID, MIRBuilder))2204    return true;2205 2206  switch (ID) {2207  default:2208    break;2209  case Intrinsic::lifetime_start:2210  case Intrinsic::lifetime_end: {2211    // No stack colouring in O0, discard region information.2212    if (MF->getTarget().getOptLevel() == CodeGenOptLevel::None ||2213        MF->getFunction().hasOptNone())2214      return true;2215 2216    unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START2217                                                  : TargetOpcode::LIFETIME_END;2218 2219    const AllocaInst *AI = dyn_cast<AllocaInst>(CI.getArgOperand(0));2220    if (!AI || !AI->isStaticAlloca())2221      return true;2222 2223    MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));2224    return true;2225  }2226  case Intrinsic::fake_use: {2227    SmallVector<llvm::SrcOp, 4> VRegs;2228    for (const auto &Arg : CI.args())2229      llvm::append_range(VRegs, getOrCreateVRegs(*Arg));2230    MIRBuilder.buildInstr(TargetOpcode::FAKE_USE, {}, VRegs);2231    MF->setHasFakeUses(true);2232    return true;2233  }2234  case Intrinsic::dbg_declare: {2235    const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);2236    assert(DI.getVariable() && "Missing variable");2237    translateDbgDeclareRecord(DI.getAddress(), DI.hasArgList(), DI.getVariable(),2238                       DI.getExpression(), DI.getDebugLoc(), MIRBuilder);2239    return true;2240  }2241  case Intrinsic::dbg_label: {2242    const DbgLabelInst &DI = cast<DbgLabelInst>(CI);2243    assert(DI.getLabel() && "Missing label");2244 2245    assert(DI.getLabel()->isValidLocationForIntrinsic(2246               MIRBuilder.getDebugLoc()) &&2247           "Expected inlined-at fields to agree");2248 2249    MIRBuilder.buildDbgLabel(DI.getLabel());2250    return true;2251  }2252  case Intrinsic::vaend:2253    // No target I know of cares about va_end. Certainly no in-tree target2254    // does. Simplest intrinsic ever!2255    return true;2256  case Intrinsic::vastart: {2257    Value *Ptr = CI.getArgOperand(0);2258    unsigned ListSize = TLI->getVaListSizeInBits(*DL) / 8;2259    Align Alignment = getKnownAlignment(Ptr, *DL);2260 2261    MIRBuilder.buildInstr(TargetOpcode::G_VASTART, {}, {getOrCreateVReg(*Ptr)})2262        .addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Ptr),2263                                                MachineMemOperand::MOStore,2264                                                ListSize, Alignment));2265    return true;2266  }2267  case Intrinsic::dbg_assign:2268    // A dbg.assign is a dbg.value with more information about stack locations,2269    // typically produced during optimisation of variables with leaked2270    // addresses. We can treat it like a normal dbg_value intrinsic here; to2271    // benefit from the full analysis of stack/SSA locations, GlobalISel would2272    // need to register for and use the AssignmentTrackingAnalysis pass.2273    [[fallthrough]];2274  case Intrinsic::dbg_value: {2275    // This form of DBG_VALUE is target-independent.2276    const DbgValueInst &DI = cast<DbgValueInst>(CI);2277    translateDbgValueRecord(DI.getValue(), DI.hasArgList(), DI.getVariable(),2278                       DI.getExpression(), DI.getDebugLoc(), MIRBuilder);2279    return true;2280  }2281  case Intrinsic::uadd_with_overflow:2282    return translateOverflowIntrinsic(CI, TargetOpcode::G_UADDO, MIRBuilder);2283  case Intrinsic::sadd_with_overflow:2284    return translateOverflowIntrinsic(CI, TargetOpcode::G_SADDO, MIRBuilder);2285  case Intrinsic::usub_with_overflow:2286    return translateOverflowIntrinsic(CI, TargetOpcode::G_USUBO, MIRBuilder);2287  case Intrinsic::ssub_with_overflow:2288    return translateOverflowIntrinsic(CI, TargetOpcode::G_SSUBO, MIRBuilder);2289  case Intrinsic::umul_with_overflow:2290    return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);2291  case Intrinsic::smul_with_overflow:2292    return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);2293  case Intrinsic::uadd_sat:2294    return translateBinaryOp(TargetOpcode::G_UADDSAT, CI, MIRBuilder);2295  case Intrinsic::sadd_sat:2296    return translateBinaryOp(TargetOpcode::G_SADDSAT, CI, MIRBuilder);2297  case Intrinsic::usub_sat:2298    return translateBinaryOp(TargetOpcode::G_USUBSAT, CI, MIRBuilder);2299  case Intrinsic::ssub_sat:2300    return translateBinaryOp(TargetOpcode::G_SSUBSAT, CI, MIRBuilder);2301  case Intrinsic::ushl_sat:2302    return translateBinaryOp(TargetOpcode::G_USHLSAT, CI, MIRBuilder);2303  case Intrinsic::sshl_sat:2304    return translateBinaryOp(TargetOpcode::G_SSHLSAT, CI, MIRBuilder);2305  case Intrinsic::umin:2306    return translateBinaryOp(TargetOpcode::G_UMIN, CI, MIRBuilder);2307  case Intrinsic::umax:2308    return translateBinaryOp(TargetOpcode::G_UMAX, CI, MIRBuilder);2309  case Intrinsic::smin:2310    return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);2311  case Intrinsic::smax:2312    return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);2313  case Intrinsic::abs:2314    // TODO: Preserve "int min is poison" arg in GMIR?2315    return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);2316  case Intrinsic::smul_fix:2317    return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);2318  case Intrinsic::umul_fix:2319    return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);2320  case Intrinsic::smul_fix_sat:2321    return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);2322  case Intrinsic::umul_fix_sat:2323    return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);2324  case Intrinsic::sdiv_fix:2325    return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);2326  case Intrinsic::udiv_fix:2327    return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);2328  case Intrinsic::sdiv_fix_sat:2329    return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);2330  case Intrinsic::udiv_fix_sat:2331    return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);2332  case Intrinsic::fmuladd: {2333    const TargetMachine &TM = MF->getTarget();2334    Register Dst = getOrCreateVReg(CI);2335    Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));2336    Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));2337    Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));2338    if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&2339        TLI->isFMAFasterThanFMulAndFAdd(*MF,2340                                        TLI->getValueType(*DL, CI.getType()))) {2341      // TODO: Revisit this to see if we should move this part of the2342      // lowering to the combiner.2343      MIRBuilder.buildFMA(Dst, Op0, Op1, Op2,2344                          MachineInstr::copyFlagsFromInstruction(CI));2345    } else {2346      LLT Ty = getLLTForType(*CI.getType(), *DL);2347      auto FMul = MIRBuilder.buildFMul(2348          Ty, Op0, Op1, MachineInstr::copyFlagsFromInstruction(CI));2349      MIRBuilder.buildFAdd(Dst, FMul, Op2,2350                           MachineInstr::copyFlagsFromInstruction(CI));2351    }2352    return true;2353  }2354  case Intrinsic::convert_from_fp16:2355    // FIXME: This intrinsic should probably be removed from the IR.2356    MIRBuilder.buildFPExt(getOrCreateVReg(CI),2357                          getOrCreateVReg(*CI.getArgOperand(0)),2358                          MachineInstr::copyFlagsFromInstruction(CI));2359    return true;2360  case Intrinsic::convert_to_fp16:2361    // FIXME: This intrinsic should probably be removed from the IR.2362    MIRBuilder.buildFPTrunc(getOrCreateVReg(CI),2363                            getOrCreateVReg(*CI.getArgOperand(0)),2364                            MachineInstr::copyFlagsFromInstruction(CI));2365    return true;2366  case Intrinsic::frexp: {2367    ArrayRef<Register> VRegs = getOrCreateVRegs(CI);2368    MIRBuilder.buildFFrexp(VRegs[0], VRegs[1],2369                           getOrCreateVReg(*CI.getArgOperand(0)),2370                           MachineInstr::copyFlagsFromInstruction(CI));2371    return true;2372  }2373  case Intrinsic::modf: {2374    ArrayRef<Register> VRegs = getOrCreateVRegs(CI);2375    MIRBuilder.buildModf(VRegs[0], VRegs[1],2376                         getOrCreateVReg(*CI.getArgOperand(0)),2377                         MachineInstr::copyFlagsFromInstruction(CI));2378    return true;2379  }2380  case Intrinsic::sincos: {2381    ArrayRef<Register> VRegs = getOrCreateVRegs(CI);2382    MIRBuilder.buildFSincos(VRegs[0], VRegs[1],2383                            getOrCreateVReg(*CI.getArgOperand(0)),2384                            MachineInstr::copyFlagsFromInstruction(CI));2385    return true;2386  }2387  case Intrinsic::fptosi_sat:2388    MIRBuilder.buildFPTOSI_SAT(getOrCreateVReg(CI),2389                               getOrCreateVReg(*CI.getArgOperand(0)));2390    return true;2391  case Intrinsic::fptoui_sat:2392    MIRBuilder.buildFPTOUI_SAT(getOrCreateVReg(CI),2393                               getOrCreateVReg(*CI.getArgOperand(0)));2394    return true;2395  case Intrinsic::memcpy_inline:2396    return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY_INLINE);2397  case Intrinsic::memcpy:2398    return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMCPY);2399  case Intrinsic::memmove:2400    return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMMOVE);2401  case Intrinsic::memset:2402    return translateMemFunc(CI, MIRBuilder, TargetOpcode::G_MEMSET);2403  case Intrinsic::eh_typeid_for: {2404    GlobalValue *GV = ExtractTypeInfo(CI.getArgOperand(0));2405    Register Reg = getOrCreateVReg(CI);2406    unsigned TypeID = MF->getTypeIDFor(GV);2407    MIRBuilder.buildConstant(Reg, TypeID);2408    return true;2409  }2410  case Intrinsic::objectsize:2411    llvm_unreachable("llvm.objectsize.* should have been lowered already");2412 2413  case Intrinsic::is_constant:2414    llvm_unreachable("llvm.is.constant.* should have been lowered already");2415 2416  case Intrinsic::stackguard:2417    getStackGuard(getOrCreateVReg(CI), MIRBuilder);2418    return true;2419  case Intrinsic::stackprotector: {2420    LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);2421    Register GuardVal;2422    if (TLI->useLoadStackGuardNode(*CI.getModule())) {2423      GuardVal = MRI->createGenericVirtualRegister(PtrTy);2424      getStackGuard(GuardVal, MIRBuilder);2425    } else2426      GuardVal = getOrCreateVReg(*CI.getArgOperand(0)); // The guard's value.2427 2428    AllocaInst *Slot = cast<AllocaInst>(CI.getArgOperand(1));2429    int FI = getOrCreateFrameIndex(*Slot);2430    MF->getFrameInfo().setStackProtectorIndex(FI);2431 2432    MIRBuilder.buildStore(2433        GuardVal, getOrCreateVReg(*Slot),2434        *MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),2435                                  MachineMemOperand::MOStore |2436                                      MachineMemOperand::MOVolatile,2437                                  PtrTy, Align(8)));2438    return true;2439  }2440  case Intrinsic::stacksave: {2441    MIRBuilder.buildInstr(TargetOpcode::G_STACKSAVE, {getOrCreateVReg(CI)}, {});2442    return true;2443  }2444  case Intrinsic::stackrestore: {2445    MIRBuilder.buildInstr(TargetOpcode::G_STACKRESTORE, {},2446                          {getOrCreateVReg(*CI.getArgOperand(0))});2447    return true;2448  }2449  case Intrinsic::cttz:2450  case Intrinsic::ctlz: {2451    ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));2452    bool isTrailing = ID == Intrinsic::cttz;2453    unsigned Opcode = isTrailing2454                          ? Cst->isZero() ? TargetOpcode::G_CTTZ2455                                          : TargetOpcode::G_CTTZ_ZERO_UNDEF2456                          : Cst->isZero() ? TargetOpcode::G_CTLZ2457                                          : TargetOpcode::G_CTLZ_ZERO_UNDEF;2458    MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},2459                          {getOrCreateVReg(*CI.getArgOperand(0))});2460    return true;2461  }2462  case Intrinsic::invariant_start: {2463    MIRBuilder.buildUndef(getOrCreateVReg(CI));2464    return true;2465  }2466  case Intrinsic::invariant_end:2467    return true;2468  case Intrinsic::expect:2469  case Intrinsic::expect_with_probability:2470  case Intrinsic::annotation:2471  case Intrinsic::ptr_annotation:2472  case Intrinsic::launder_invariant_group:2473  case Intrinsic::strip_invariant_group: {2474    // Drop the intrinsic, but forward the value.2475    MIRBuilder.buildCopy(getOrCreateVReg(CI),2476                         getOrCreateVReg(*CI.getArgOperand(0)));2477    return true;2478  }2479  case Intrinsic::assume:2480  case Intrinsic::experimental_noalias_scope_decl:2481  case Intrinsic::var_annotation:2482  case Intrinsic::sideeffect:2483    // Discard annotate attributes, assumptions, and artificial side-effects.2484    return true;2485  case Intrinsic::read_volatile_register:2486  case Intrinsic::read_register: {2487    Value *Arg = CI.getArgOperand(0);2488    MIRBuilder2489        .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})2490        .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));2491    return true;2492  }2493  case Intrinsic::write_register: {2494    Value *Arg = CI.getArgOperand(0);2495    MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)2496      .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))2497      .addUse(getOrCreateVReg(*CI.getArgOperand(1)));2498    return true;2499  }2500  case Intrinsic::localescape: {2501    MachineBasicBlock &EntryMBB = MF->front();2502    StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(MF->getName());2503 2504    // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission2505    // is the same on all targets.2506    for (unsigned Idx = 0, E = CI.arg_size(); Idx < E; ++Idx) {2507      Value *Arg = CI.getArgOperand(Idx)->stripPointerCasts();2508      if (isa<ConstantPointerNull>(Arg))2509        continue; // Skip null pointers. They represent a hole in index space.2510 2511      int FI = getOrCreateFrameIndex(*cast<AllocaInst>(Arg));2512      MCSymbol *FrameAllocSym =2513          MF->getContext().getOrCreateFrameAllocSymbol(EscapedName, Idx);2514 2515      // This should be inserted at the start of the entry block.2516      auto LocalEscape =2517          MIRBuilder.buildInstrNoInsert(TargetOpcode::LOCAL_ESCAPE)2518              .addSym(FrameAllocSym)2519              .addFrameIndex(FI);2520 2521      EntryMBB.insert(EntryMBB.begin(), LocalEscape);2522    }2523 2524    return true;2525  }2526  case Intrinsic::vector_reduce_fadd:2527  case Intrinsic::vector_reduce_fmul: {2528    // Need to check for the reassoc flag to decide whether we want a2529    // sequential reduction opcode or not.2530    Register Dst = getOrCreateVReg(CI);2531    Register ScalarSrc = getOrCreateVReg(*CI.getArgOperand(0));2532    Register VecSrc = getOrCreateVReg(*CI.getArgOperand(1));2533    unsigned Opc = 0;2534    if (!CI.hasAllowReassoc()) {2535      // The sequential ordering case.2536      Opc = ID == Intrinsic::vector_reduce_fadd2537                ? TargetOpcode::G_VECREDUCE_SEQ_FADD2538                : TargetOpcode::G_VECREDUCE_SEQ_FMUL;2539      if (!MRI->getType(VecSrc).isVector())2540        Opc = ID == Intrinsic::vector_reduce_fadd ? TargetOpcode::G_FADD2541                                                  : TargetOpcode::G_FMUL;2542      MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},2543                            MachineInstr::copyFlagsFromInstruction(CI));2544      return true;2545    }2546    // We split the operation into a separate G_FADD/G_FMUL + the reduce,2547    // since the associativity doesn't matter.2548    unsigned ScalarOpc;2549    if (ID == Intrinsic::vector_reduce_fadd) {2550      Opc = TargetOpcode::G_VECREDUCE_FADD;2551      ScalarOpc = TargetOpcode::G_FADD;2552    } else {2553      Opc = TargetOpcode::G_VECREDUCE_FMUL;2554      ScalarOpc = TargetOpcode::G_FMUL;2555    }2556    LLT DstTy = MRI->getType(Dst);2557    auto Rdx = MIRBuilder.buildInstr(2558        Opc, {DstTy}, {VecSrc}, MachineInstr::copyFlagsFromInstruction(CI));2559    MIRBuilder.buildInstr(ScalarOpc, {Dst}, {ScalarSrc, Rdx},2560                          MachineInstr::copyFlagsFromInstruction(CI));2561 2562    return true;2563  }2564  case Intrinsic::trap:2565    return translateTrap(CI, MIRBuilder, TargetOpcode::G_TRAP);2566  case Intrinsic::debugtrap:2567    return translateTrap(CI, MIRBuilder, TargetOpcode::G_DEBUGTRAP);2568  case Intrinsic::ubsantrap:2569    return translateTrap(CI, MIRBuilder, TargetOpcode::G_UBSANTRAP);2570  case Intrinsic::allow_runtime_check:2571  case Intrinsic::allow_ubsan_check:2572    MIRBuilder.buildCopy(getOrCreateVReg(CI),2573                         getOrCreateVReg(*ConstantInt::getTrue(CI.getType())));2574    return true;2575  case Intrinsic::amdgcn_cs_chain:2576  case Intrinsic::amdgcn_call_whole_wave:2577    return translateCallBase(CI, MIRBuilder);2578  case Intrinsic::fptrunc_round: {2579    uint32_t Flags = MachineInstr::copyFlagsFromInstruction(CI);2580 2581    // Convert the metadata argument to a constant integer2582    Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(1))->getMetadata();2583    std::optional<RoundingMode> RoundMode =2584        convertStrToRoundingMode(cast<MDString>(MD)->getString());2585 2586    // Add the Rounding mode as an integer2587    MIRBuilder2588        .buildInstr(TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND,2589                    {getOrCreateVReg(CI)},2590                    {getOrCreateVReg(*CI.getArgOperand(0))}, Flags)2591        .addImm((int)*RoundMode);2592 2593    return true;2594  }2595  case Intrinsic::is_fpclass: {2596    Value *FpValue = CI.getOperand(0);2597    ConstantInt *TestMaskValue = cast<ConstantInt>(CI.getOperand(1));2598 2599    MIRBuilder2600        .buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)},2601                    {getOrCreateVReg(*FpValue)})2602        .addImm(TestMaskValue->getZExtValue());2603 2604    return true;2605  }2606  case Intrinsic::set_fpenv: {2607    Value *FPEnv = CI.getOperand(0);2608    MIRBuilder.buildSetFPEnv(getOrCreateVReg(*FPEnv));2609    return true;2610  }2611  case Intrinsic::reset_fpenv:2612    MIRBuilder.buildResetFPEnv();2613    return true;2614  case Intrinsic::set_fpmode: {2615    Value *FPState = CI.getOperand(0);2616    MIRBuilder.buildSetFPMode(getOrCreateVReg(*FPState));2617    return true;2618  }2619  case Intrinsic::reset_fpmode:2620    MIRBuilder.buildResetFPMode();2621    return true;2622  case Intrinsic::get_rounding:2623    MIRBuilder.buildGetRounding(getOrCreateVReg(CI));2624    return true;2625  case Intrinsic::set_rounding:2626    MIRBuilder.buildSetRounding(getOrCreateVReg(*CI.getOperand(0)));2627    return true;2628  case Intrinsic::vscale: {2629    MIRBuilder.buildVScale(getOrCreateVReg(CI), 1);2630    return true;2631  }2632  case Intrinsic::scmp:2633    MIRBuilder.buildSCmp(getOrCreateVReg(CI),2634                         getOrCreateVReg(*CI.getOperand(0)),2635                         getOrCreateVReg(*CI.getOperand(1)));2636    return true;2637  case Intrinsic::ucmp:2638    MIRBuilder.buildUCmp(getOrCreateVReg(CI),2639                         getOrCreateVReg(*CI.getOperand(0)),2640                         getOrCreateVReg(*CI.getOperand(1)));2641    return true;2642  case Intrinsic::vector_extract:2643    return translateExtractVector(CI, MIRBuilder);2644  case Intrinsic::vector_insert:2645    return translateInsertVector(CI, MIRBuilder);2646  case Intrinsic::stepvector: {2647    MIRBuilder.buildStepVector(getOrCreateVReg(CI), 1);2648    return true;2649  }2650  case Intrinsic::prefetch: {2651    Value *Addr = CI.getOperand(0);2652    unsigned RW = cast<ConstantInt>(CI.getOperand(1))->getZExtValue();2653    unsigned Locality = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();2654    unsigned CacheType = cast<ConstantInt>(CI.getOperand(3))->getZExtValue();2655 2656    auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;2657    auto &MMO = *MF->getMachineMemOperand(MachinePointerInfo(Addr), Flags,2658                                          LLT(), Align());2659 2660    MIRBuilder.buildPrefetch(getOrCreateVReg(*Addr), RW, Locality, CacheType,2661                             MMO);2662 2663    return true;2664  }2665 2666  case Intrinsic::vector_interleave2:2667  case Intrinsic::vector_deinterleave2: {2668    // Both intrinsics have at least one operand.2669    Value *Op0 = CI.getOperand(0);2670    LLT ResTy = getLLTForType(*Op0->getType(), MIRBuilder.getDataLayout());2671    if (!ResTy.isFixedVector())2672      return false;2673 2674    if (CI.getIntrinsicID() == Intrinsic::vector_interleave2)2675      return translateVectorInterleave2Intrinsic(CI, MIRBuilder);2676 2677    return translateVectorDeinterleave2Intrinsic(CI, MIRBuilder);2678  }2679 2680#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)  \2681  case Intrinsic::INTRINSIC:2682#include "llvm/IR/ConstrainedOps.def"2683    return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),2684                                           MIRBuilder);2685  case Intrinsic::experimental_convergence_anchor:2686  case Intrinsic::experimental_convergence_entry:2687  case Intrinsic::experimental_convergence_loop:2688    return translateConvergenceControlIntrinsic(CI, ID, MIRBuilder);2689  case Intrinsic::reloc_none: {2690    Metadata *MD = cast<MetadataAsValue>(CI.getArgOperand(0))->getMetadata();2691    StringRef SymbolName = cast<MDString>(MD)->getString();2692    MIRBuilder.buildInstr(TargetOpcode::RELOC_NONE)2693        .addExternalSymbol(SymbolName.data());2694    return true;2695  }2696  }2697  return false;2698}2699 2700bool IRTranslator::translateInlineAsm(const CallBase &CB,2701                                      MachineIRBuilder &MIRBuilder) {2702  if (containsBF16Type(CB) && !targetSupportsBF16Type(MF))2703    return false;2704 2705  const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();2706 2707  if (!ALI) {2708    LLVM_DEBUG(2709        dbgs() << "Inline asm lowering is not supported for this target yet\n");2710    return false;2711  }2712 2713  return ALI->lowerInlineAsm(2714      MIRBuilder, CB, [&](const Value &Val) { return getOrCreateVRegs(Val); });2715}2716 2717bool IRTranslator::translateCallBase(const CallBase &CB,2718                                     MachineIRBuilder &MIRBuilder) {2719  ArrayRef<Register> Res = getOrCreateVRegs(CB);2720 2721  SmallVector<ArrayRef<Register>, 8> Args;2722  Register SwiftInVReg = 0;2723  Register SwiftErrorVReg = 0;2724  for (const auto &Arg : CB.args()) {2725    if (CLI->supportSwiftError() && isSwiftError(Arg)) {2726      assert(SwiftInVReg == 0 && "Expected only one swift error argument");2727      LLT Ty = getLLTForType(*Arg->getType(), *DL);2728      SwiftInVReg = MRI->createGenericVirtualRegister(Ty);2729      MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt(2730                                            &CB, &MIRBuilder.getMBB(), Arg));2731      Args.emplace_back(ArrayRef(SwiftInVReg));2732      SwiftErrorVReg =2733          SwiftError.getOrCreateVRegDefAt(&CB, &MIRBuilder.getMBB(), Arg);2734      continue;2735    }2736    Args.push_back(getOrCreateVRegs(*Arg));2737  }2738 2739  if (auto *CI = dyn_cast<CallInst>(&CB)) {2740    if (ORE->enabled()) {2741      if (MemoryOpRemark::canHandle(CI, *LibInfo)) {2742        MemoryOpRemark R(*ORE, "gisel-irtranslator-memsize", *DL, *LibInfo);2743        R.visit(CI);2744      }2745    }2746  }2747 2748  std::optional<CallLowering::PtrAuthInfo> PAI;2749  if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_ptrauth)) {2750    // Functions should never be ptrauth-called directly.2751    assert(!CB.getCalledFunction() && "invalid direct ptrauth call");2752 2753    const Value *Key = Bundle->Inputs[0];2754    const Value *Discriminator = Bundle->Inputs[1];2755 2756    // Look through ptrauth constants to try to eliminate the matching bundle2757    // and turn this into a direct call with no ptrauth.2758    // CallLowering will use the raw pointer if it doesn't find the PAI.2759    const auto *CalleeCPA = dyn_cast<ConstantPtrAuth>(CB.getCalledOperand());2760    if (!CalleeCPA || !isa<Function>(CalleeCPA->getPointer()) ||2761        !CalleeCPA->isKnownCompatibleWith(Key, Discriminator, *DL)) {2762      // If we can't make it direct, package the bundle into PAI.2763      Register DiscReg = getOrCreateVReg(*Discriminator);2764      PAI = CallLowering::PtrAuthInfo{cast<ConstantInt>(Key)->getZExtValue(),2765                                      DiscReg};2766    }2767  }2768 2769  Register ConvergenceCtrlToken = 0;2770  if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {2771    const auto &Token = *Bundle->Inputs[0].get();2772    ConvergenceCtrlToken = getOrCreateConvergenceTokenVReg(Token);2773  }2774 2775  // We don't set HasCalls on MFI here yet because call lowering may decide to2776  // optimize into tail calls. Instead, we defer that to selection where a final2777  // scan is done to check if any instructions are calls.2778  bool Success = CLI->lowerCall(2779      MIRBuilder, CB, Res, Args, SwiftErrorVReg, PAI, ConvergenceCtrlToken,2780      [&]() { return getOrCreateVReg(*CB.getCalledOperand()); });2781 2782  // Check if we just inserted a tail call.2783  if (Success) {2784    assert(!HasTailCall && "Can't tail call return twice from block?");2785    const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();2786    HasTailCall = TII->isTailCall(*std::prev(MIRBuilder.getInsertPt()));2787  }2788 2789  return Success;2790}2791 2792bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {2793  if (containsBF16Type(U) && !targetSupportsBF16Type(MF))2794    return false;2795 2796  const CallInst &CI = cast<CallInst>(U);2797  const Function *F = CI.getCalledFunction();2798 2799  // FIXME: support Windows dllimport function calls and calls through2800  // weak symbols.2801  if (F && (F->hasDLLImportStorageClass() ||2802            (MF->getTarget().getTargetTriple().isOSWindows() &&2803             F->hasExternalWeakLinkage())))2804    return false;2805 2806  // FIXME: support control flow guard targets.2807  if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))2808    return false;2809 2810  // FIXME: support statepoints and related.2811  if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))2812    return false;2813 2814  if (CI.isInlineAsm())2815    return translateInlineAsm(CI, MIRBuilder);2816 2817  Intrinsic::ID ID = F ? F->getIntrinsicID() : Intrinsic::not_intrinsic;2818  if (!F || ID == Intrinsic::not_intrinsic) {2819    if (translateCallBase(CI, MIRBuilder)) {2820      diagnoseDontCall(CI);2821      return true;2822    }2823    return false;2824  }2825 2826  assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");2827 2828  if (translateKnownIntrinsic(CI, ID, MIRBuilder))2829    return true;2830 2831  TargetLowering::IntrinsicInfo Info;2832  bool IsTgtMemIntrinsic = TLI->getTgtMemIntrinsic(Info, CI, *MF, ID);2833 2834  return translateIntrinsic(CI, ID, MIRBuilder,2835                            IsTgtMemIntrinsic ? &Info : nullptr);2836}2837 2838/// Translate a call to an intrinsic.2839/// Depending on whether TLI->getTgtMemIntrinsic() is true, TgtMemIntrinsicInfo2840/// is a pointer to the correspondingly populated IntrinsicInfo object.2841/// Otherwise, this pointer is null.2842bool IRTranslator::translateIntrinsic(2843    const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,2844    const TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {2845  ArrayRef<Register> ResultRegs;2846  if (!CB.getType()->isVoidTy())2847    ResultRegs = getOrCreateVRegs(CB);2848 2849  // Ignore the callsite attributes. Backend code is most likely not expecting2850  // an intrinsic to sometimes have side effects and sometimes not.2851  MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs);2852  if (isa<FPMathOperator>(CB))2853    MIB->copyIRFlags(CB);2854 2855  for (const auto &Arg : enumerate(CB.args())) {2856    // If this is required to be an immediate, don't materialize it in a2857    // register.2858    if (CB.paramHasAttr(Arg.index(), Attribute::ImmArg)) {2859      if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {2860        // imm arguments are more convenient than cimm (and realistically2861        // probably sufficient), so use them.2862        assert(CI->getBitWidth() <= 64 &&2863               "large intrinsic immediates not handled");2864        MIB.addImm(CI->getSExtValue());2865      } else {2866        MIB.addFPImm(cast<ConstantFP>(Arg.value()));2867      }2868    } else if (auto *MDVal = dyn_cast<MetadataAsValue>(Arg.value())) {2869      auto *MD = MDVal->getMetadata();2870      auto *MDN = dyn_cast<MDNode>(MD);2871      if (!MDN) {2872        if (auto *ConstMD = dyn_cast<ConstantAsMetadata>(MD))2873          MDN = MDNode::get(MF->getFunction().getContext(), ConstMD);2874        else // This was probably an MDString.2875          return false;2876      }2877      MIB.addMetadata(MDN);2878    } else {2879      ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());2880      if (VRegs.size() > 1)2881        return false;2882      MIB.addUse(VRegs[0]);2883    }2884  }2885 2886  // Add a MachineMemOperand if it is a target mem intrinsic.2887  if (TgtMemIntrinsicInfo) {2888    const Function *F = CB.getCalledFunction();2889 2890    Align Alignment = TgtMemIntrinsicInfo->align.value_or(DL->getABITypeAlign(2891        TgtMemIntrinsicInfo->memVT.getTypeForEVT(F->getContext())));2892    LLT MemTy =2893        TgtMemIntrinsicInfo->memVT.isSimple()2894            ? getLLTForMVT(TgtMemIntrinsicInfo->memVT.getSimpleVT())2895            : LLT::scalar(TgtMemIntrinsicInfo->memVT.getStoreSizeInBits());2896 2897    // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic2898    //       didn't yield anything useful.2899    MachinePointerInfo MPI;2900    if (TgtMemIntrinsicInfo->ptrVal) {2901      MPI = MachinePointerInfo(TgtMemIntrinsicInfo->ptrVal,2902                               TgtMemIntrinsicInfo->offset);2903    } else if (TgtMemIntrinsicInfo->fallbackAddressSpace) {2904      MPI = MachinePointerInfo(*TgtMemIntrinsicInfo->fallbackAddressSpace);2905    }2906    MIB.addMemOperand(MF->getMachineMemOperand(2907        MPI, TgtMemIntrinsicInfo->flags, MemTy, Alignment, CB.getAAMetadata(),2908        /*Ranges=*/nullptr, TgtMemIntrinsicInfo->ssid,2909        TgtMemIntrinsicInfo->order, TgtMemIntrinsicInfo->failureOrder));2910  }2911 2912  if (CB.isConvergent()) {2913    if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {2914      auto *Token = Bundle->Inputs[0].get();2915      Register TokenReg = getOrCreateVReg(*Token);2916      MIB.addUse(TokenReg, RegState::Implicit);2917    }2918  }2919 2920  if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_deactivation_symbol))2921    MIB->setDeactivationSymbol(*MF, Bundle->Inputs[0].get());2922 2923  return true;2924}2925 2926bool IRTranslator::findUnwindDestinations(2927    const BasicBlock *EHPadBB,2928    BranchProbability Prob,2929    SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>2930        &UnwindDests) {2931  EHPersonality Personality = classifyEHPersonality(2932      EHPadBB->getParent()->getFunction().getPersonalityFn());2933  bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;2934  bool IsCoreCLR = Personality == EHPersonality::CoreCLR;2935  bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;2936  bool IsSEH = isAsynchronousEHPersonality(Personality);2937 2938  if (IsWasmCXX) {2939    // Ignore this for now.2940    return false;2941  }2942 2943  while (EHPadBB) {2944    BasicBlock::const_iterator Pad = EHPadBB->getFirstNonPHIIt();2945    BasicBlock *NewEHPadBB = nullptr;2946    if (isa<LandingPadInst>(Pad)) {2947      // Stop on landingpads. They are not funclets.2948      UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);2949      break;2950    }2951    if (isa<CleanupPadInst>(Pad)) {2952      // Stop on cleanup pads. Cleanups are always funclet entries for all known2953      // personalities.2954      UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob);2955      UnwindDests.back().first->setIsEHScopeEntry();2956      UnwindDests.back().first->setIsEHFuncletEntry();2957      break;2958    }2959    if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {2960      // Add the catchpad handlers to the possible destinations.2961      for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {2962        UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob);2963        // For MSVC++ and the CLR, catchblocks are funclets and need prologues.2964        if (IsMSVCCXX || IsCoreCLR)2965          UnwindDests.back().first->setIsEHFuncletEntry();2966        if (!IsSEH)2967          UnwindDests.back().first->setIsEHScopeEntry();2968      }2969      NewEHPadBB = CatchSwitch->getUnwindDest();2970    } else {2971      continue;2972    }2973 2974    BranchProbabilityInfo *BPI = FuncInfo.BPI;2975    if (BPI && NewEHPadBB)2976      Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);2977    EHPadBB = NewEHPadBB;2978  }2979  return true;2980}2981 2982bool IRTranslator::translateInvoke(const User &U,2983                                   MachineIRBuilder &MIRBuilder) {2984  const InvokeInst &I = cast<InvokeInst>(U);2985  MCContext &Context = MF->getContext();2986 2987  const BasicBlock *ReturnBB = I.getSuccessor(0);2988  const BasicBlock *EHPadBB = I.getSuccessor(1);2989 2990  const Function *Fn = I.getCalledFunction();2991 2992  // FIXME: support invoking patchpoint and statepoint intrinsics.2993  if (Fn && Fn->isIntrinsic())2994    return false;2995 2996  // FIXME: support whatever these are.2997  if (I.hasDeoptState())2998    return false;2999 3000  // FIXME: support control flow guard targets.3001  if (I.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))3002    return false;3003 3004  // FIXME: support Windows exception handling.3005  if (!isa<LandingPadInst>(EHPadBB->getFirstNonPHIIt()))3006    return false;3007 3008  // FIXME: support Windows dllimport function calls and calls through3009  // weak symbols.3010  if (Fn && (Fn->hasDLLImportStorageClass() ||3011            (MF->getTarget().getTargetTriple().isOSWindows() &&3012             Fn->hasExternalWeakLinkage())))3013    return false;3014 3015  bool LowerInlineAsm = I.isInlineAsm();3016  bool NeedEHLabel = true;3017 3018  // Emit the actual call, bracketed by EH_LABELs so that the MF knows about3019  // the region covered by the try.3020  MCSymbol *BeginSymbol = nullptr;3021  if (NeedEHLabel) {3022    MIRBuilder.buildInstr(TargetOpcode::G_INVOKE_REGION_START);3023    BeginSymbol = Context.createTempSymbol();3024    MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);3025  }3026 3027  if (LowerInlineAsm) {3028    if (!translateInlineAsm(I, MIRBuilder))3029      return false;3030  } else if (!translateCallBase(I, MIRBuilder))3031    return false;3032 3033  MCSymbol *EndSymbol = nullptr;3034  if (NeedEHLabel) {3035    EndSymbol = Context.createTempSymbol();3036    MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);3037  }3038 3039  SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;3040  BranchProbabilityInfo *BPI = FuncInfo.BPI;3041  MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB();3042  BranchProbability EHPadBBProb =3043      BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)3044          : BranchProbability::getZero();3045 3046  if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests))3047    return false;3048 3049  MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),3050                    &ReturnMBB = getMBB(*ReturnBB);3051  // Update successor info.3052  addSuccessorWithProb(InvokeMBB, &ReturnMBB);3053  for (auto &UnwindDest : UnwindDests) {3054    UnwindDest.first->setIsEHPad();3055    addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);3056  }3057  InvokeMBB->normalizeSuccProbs();3058 3059  if (NeedEHLabel) {3060    assert(BeginSymbol && "Expected a begin symbol!");3061    assert(EndSymbol && "Expected an end symbol!");3062    MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);3063  }3064 3065  MIRBuilder.buildBr(ReturnMBB);3066  return true;3067}3068 3069bool IRTranslator::translateCallBr(const User &U,3070                                   MachineIRBuilder &MIRBuilder) {3071  // FIXME: Implement this.3072  return false;3073}3074 3075bool IRTranslator::translateLandingPad(const User &U,3076                                       MachineIRBuilder &MIRBuilder) {3077  const LandingPadInst &LP = cast<LandingPadInst>(U);3078 3079  MachineBasicBlock &MBB = MIRBuilder.getMBB();3080 3081  MBB.setIsEHPad();3082 3083  // If there aren't registers to copy the values into (e.g., during SjLj3084  // exceptions), then don't bother.3085  const Constant *PersonalityFn = MF->getFunction().getPersonalityFn();3086  if (TLI->getExceptionPointerRegister(PersonalityFn) == 0 &&3087      TLI->getExceptionSelectorRegister(PersonalityFn) == 0)3088    return true;3089 3090  // If landingpad's return type is token type, we don't create DAG nodes3091  // for its exception pointer and selector value. The extraction of exception3092  // pointer or selector value from token type landingpads is not currently3093  // supported.3094  if (LP.getType()->isTokenTy())3095    return true;3096 3097  // Add a label to mark the beginning of the landing pad.  Deletion of the3098  // landing pad can thus be detected via the MachineModuleInfo.3099  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)3100    .addSym(MF->addLandingPad(&MBB));3101 3102  // If the unwinder does not preserve all registers, ensure that the3103  // function marks the clobbered registers as used.3104  const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();3105  if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))3106    MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);3107 3108  LLT Ty = getLLTForType(*LP.getType(), *DL);3109  Register Undef = MRI->createGenericVirtualRegister(Ty);3110  MIRBuilder.buildUndef(Undef);3111 3112  SmallVector<LLT, 2> Tys;3113  for (Type *Ty : cast<StructType>(LP.getType())->elements())3114    Tys.push_back(getLLTForType(*Ty, *DL));3115  assert(Tys.size() == 2 && "Only two-valued landingpads are supported");3116 3117  // Mark exception register as live in.3118  Register ExceptionReg = TLI->getExceptionPointerRegister(PersonalityFn);3119  if (!ExceptionReg)3120    return false;3121 3122  MBB.addLiveIn(ExceptionReg);3123  ArrayRef<Register> ResRegs = getOrCreateVRegs(LP);3124  MIRBuilder.buildCopy(ResRegs[0], ExceptionReg);3125 3126  Register SelectorReg = TLI->getExceptionSelectorRegister(PersonalityFn);3127  if (!SelectorReg)3128    return false;3129 3130  MBB.addLiveIn(SelectorReg);3131  Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]);3132  MIRBuilder.buildCopy(PtrVReg, SelectorReg);3133  MIRBuilder.buildCast(ResRegs[1], PtrVReg);3134 3135  return true;3136}3137 3138bool IRTranslator::translateAlloca(const User &U,3139                                   MachineIRBuilder &MIRBuilder) {3140  auto &AI = cast<AllocaInst>(U);3141 3142  if (AI.isSwiftError())3143    return true;3144 3145  if (AI.isStaticAlloca()) {3146    Register Res = getOrCreateVReg(AI);3147    int FI = getOrCreateFrameIndex(AI);3148    MIRBuilder.buildFrameIndex(Res, FI);3149    return true;3150  }3151 3152  // FIXME: support stack probing for Windows.3153  if (MF->getTarget().getTargetTriple().isOSWindows())3154    return false;3155 3156  // Now we're in the harder dynamic case.3157  Register NumElts = getOrCreateVReg(*AI.getArraySize());3158  Type *IntPtrIRTy = DL->getIntPtrType(AI.getType());3159  LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL);3160  if (MRI->getType(NumElts) != IntPtrTy) {3161    Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy);3162    MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts);3163    NumElts = ExtElts;3164  }3165 3166  Type *Ty = AI.getAllocatedType();3167 3168  Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy);3169  Register TySize =3170      getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, DL->getTypeAllocSize(Ty)));3171  MIRBuilder.buildMul(AllocSize, NumElts, TySize);3172 3173  // Round the size of the allocation up to the stack alignment size3174  // by add SA-1 to the size. This doesn't overflow because we're computing3175  // an address inside an alloca.3176  Align StackAlign = MF->getSubtarget().getFrameLowering()->getStackAlign();3177  auto SAMinusOne = MIRBuilder.buildConstant(IntPtrTy, StackAlign.value() - 1);3178  auto AllocAdd = MIRBuilder.buildAdd(IntPtrTy, AllocSize, SAMinusOne,3179                                      MachineInstr::NoUWrap);3180  auto AlignCst =3181      MIRBuilder.buildConstant(IntPtrTy, ~(uint64_t)(StackAlign.value() - 1));3182  auto AlignedAlloc = MIRBuilder.buildAnd(IntPtrTy, AllocAdd, AlignCst);3183 3184  Align Alignment = std::max(AI.getAlign(), DL->getPrefTypeAlign(Ty));3185  if (Alignment <= StackAlign)3186    Alignment = Align(1);3187  MIRBuilder.buildDynStackAlloc(getOrCreateVReg(AI), AlignedAlloc, Alignment);3188 3189  MF->getFrameInfo().CreateVariableSizedObject(Alignment, &AI);3190  assert(MF->getFrameInfo().hasVarSizedObjects());3191  return true;3192}3193 3194bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {3195  // FIXME: We may need more info about the type. Because of how LLT works,3196  // we're completely discarding the i64/double distinction here (amongst3197  // others). Fortunately the ABIs I know of where that matters don't use va_arg3198  // anyway but that's not guaranteed.3199  MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},3200                        {getOrCreateVReg(*U.getOperand(0)),3201                         DL->getABITypeAlign(U.getType()).value()});3202  return true;3203}3204 3205bool IRTranslator::translateUnreachable(const User &U,3206                                        MachineIRBuilder &MIRBuilder) {3207  auto &UI = cast<UnreachableInst>(U);3208  if (!UI.shouldLowerToTrap(MF->getTarget().Options.TrapUnreachable,3209                            MF->getTarget().Options.NoTrapAfterNoreturn))3210    return true;3211 3212  MIRBuilder.buildTrap();3213  return true;3214}3215 3216bool IRTranslator::translateInsertElement(const User &U,3217                                          MachineIRBuilder &MIRBuilder) {3218  // If it is a <1 x Ty> vector, use the scalar as it is3219  // not a legal vector type in LLT.3220  if (auto *FVT = dyn_cast<FixedVectorType>(U.getType());3221      FVT && FVT->getNumElements() == 1)3222    return translateCopy(U, *U.getOperand(1), MIRBuilder);3223 3224  Register Res = getOrCreateVReg(U);3225  Register Val = getOrCreateVReg(*U.getOperand(0));3226  Register Elt = getOrCreateVReg(*U.getOperand(1));3227  unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(*DL);3228  Register Idx;3229  if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(2))) {3230    if (CI->getBitWidth() != PreferredVecIdxWidth) {3231      APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);3232      auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);3233      Idx = getOrCreateVReg(*NewIdxCI);3234    }3235  }3236  if (!Idx)3237    Idx = getOrCreateVReg(*U.getOperand(2));3238  if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {3239    const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);3240    Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);3241  }3242  MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx);3243  return true;3244}3245 3246bool IRTranslator::translateInsertVector(const User &U,3247                                         MachineIRBuilder &MIRBuilder) {3248  Register Dst = getOrCreateVReg(U);3249  Register Vec = getOrCreateVReg(*U.getOperand(0));3250  Register Elt = getOrCreateVReg(*U.getOperand(1));3251 3252  ConstantInt *CI = cast<ConstantInt>(U.getOperand(2));3253  unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(*DL);3254 3255  // Resize Index to preferred index width.3256  if (CI->getBitWidth() != PreferredVecIdxWidth) {3257    APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);3258    CI = ConstantInt::get(CI->getContext(), NewIdx);3259  }3260 3261  // If it is a <1 x Ty> vector, we have to use other means.3262  if (auto *ResultType = dyn_cast<FixedVectorType>(U.getOperand(1)->getType());3263      ResultType && ResultType->getNumElements() == 1) {3264    if (auto *InputType = dyn_cast<FixedVectorType>(U.getOperand(0)->getType());3265        InputType && InputType->getNumElements() == 1) {3266      // We are inserting an illegal fixed vector into an illegal3267      // fixed vector, use the scalar as it is not a legal vector type3268      // in LLT.3269      return translateCopy(U, *U.getOperand(0), MIRBuilder);3270    }3271    if (isa<FixedVectorType>(U.getOperand(0)->getType())) {3272      // We are inserting an illegal fixed vector into a legal fixed3273      // vector, use the scalar as it is not a legal vector type in3274      // LLT.3275      Register Idx = getOrCreateVReg(*CI);3276      MIRBuilder.buildInsertVectorElement(Dst, Vec, Elt, Idx);3277      return true;3278    }3279    if (isa<ScalableVectorType>(U.getOperand(0)->getType())) {3280      // We are inserting an illegal fixed vector into a scalable3281      // vector, use a scalar element insert.3282      LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);3283      Register Idx = getOrCreateVReg(*CI);3284      auto ScaledIndex = MIRBuilder.buildMul(3285          VecIdxTy, MIRBuilder.buildVScale(VecIdxTy, 1), Idx);3286      MIRBuilder.buildInsertVectorElement(Dst, Vec, Elt, ScaledIndex);3287      return true;3288    }3289  }3290 3291  MIRBuilder.buildInsertSubvector(3292      getOrCreateVReg(U), getOrCreateVReg(*U.getOperand(0)),3293      getOrCreateVReg(*U.getOperand(1)), CI->getZExtValue());3294  return true;3295}3296 3297bool IRTranslator::translateExtractElement(const User &U,3298                                           MachineIRBuilder &MIRBuilder) {3299  // If it is a <1 x Ty> vector, use the scalar as it is3300  // not a legal vector type in LLT.3301  if (const FixedVectorType *FVT =3302          dyn_cast<FixedVectorType>(U.getOperand(0)->getType()))3303    if (FVT->getNumElements() == 1)3304      return translateCopy(U, *U.getOperand(0), MIRBuilder);3305 3306  Register Res = getOrCreateVReg(U);3307  Register Val = getOrCreateVReg(*U.getOperand(0));3308  unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(*DL);3309  Register Idx;3310  if (auto *CI = dyn_cast<ConstantInt>(U.getOperand(1))) {3311    if (CI->getBitWidth() != PreferredVecIdxWidth) {3312      APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);3313      auto *NewIdxCI = ConstantInt::get(CI->getContext(), NewIdx);3314      Idx = getOrCreateVReg(*NewIdxCI);3315    }3316  }3317  if (!Idx)3318    Idx = getOrCreateVReg(*U.getOperand(1));3319  if (MRI->getType(Idx).getSizeInBits() != PreferredVecIdxWidth) {3320    const LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);3321    Idx = MIRBuilder.buildZExtOrTrunc(VecIdxTy, Idx).getReg(0);3322  }3323  MIRBuilder.buildExtractVectorElement(Res, Val, Idx);3324  return true;3325}3326 3327bool IRTranslator::translateExtractVector(const User &U,3328                                          MachineIRBuilder &MIRBuilder) {3329  Register Res = getOrCreateVReg(U);3330  Register Vec = getOrCreateVReg(*U.getOperand(0));3331  ConstantInt *CI = cast<ConstantInt>(U.getOperand(1));3332  unsigned PreferredVecIdxWidth = TLI->getVectorIdxWidth(*DL);3333 3334  // Resize Index to preferred index width.3335  if (CI->getBitWidth() != PreferredVecIdxWidth) {3336    APInt NewIdx = CI->getValue().zextOrTrunc(PreferredVecIdxWidth);3337    CI = ConstantInt::get(CI->getContext(), NewIdx);3338  }3339 3340  // If it is a <1 x Ty> vector, we have to use other means.3341  if (auto *ResultType = dyn_cast<FixedVectorType>(U.getType());3342      ResultType && ResultType->getNumElements() == 1) {3343    if (auto *InputType = dyn_cast<FixedVectorType>(U.getOperand(0)->getType());3344        InputType && InputType->getNumElements() == 1) {3345      // We are extracting an illegal fixed vector from an illegal fixed vector,3346      // use the scalar as it is not a legal vector type in LLT.3347      return translateCopy(U, *U.getOperand(0), MIRBuilder);3348    }3349    if (isa<FixedVectorType>(U.getOperand(0)->getType())) {3350      // We are extracting an illegal fixed vector from a legal fixed3351      // vector, use the scalar as it is not a legal vector type in3352      // LLT.3353      Register Idx = getOrCreateVReg(*CI);3354      MIRBuilder.buildExtractVectorElement(Res, Vec, Idx);3355      return true;3356    }3357    if (isa<ScalableVectorType>(U.getOperand(0)->getType())) {3358      // We are extracting an illegal fixed vector from a scalable3359      // vector, use a scalar element extract.3360      LLT VecIdxTy = LLT::scalar(PreferredVecIdxWidth);3361      Register Idx = getOrCreateVReg(*CI);3362      auto ScaledIndex = MIRBuilder.buildMul(3363          VecIdxTy, MIRBuilder.buildVScale(VecIdxTy, 1), Idx);3364      MIRBuilder.buildExtractVectorElement(Res, Vec, ScaledIndex);3365      return true;3366    }3367  }3368 3369  MIRBuilder.buildExtractSubvector(getOrCreateVReg(U),3370                                   getOrCreateVReg(*U.getOperand(0)),3371                                   CI->getZExtValue());3372  return true;3373}3374 3375bool IRTranslator::translateShuffleVector(const User &U,3376                                          MachineIRBuilder &MIRBuilder) {3377  // A ShuffleVector that operates on scalable vectors is a splat vector where3378  // the value of the splat vector is the 0th element of the first operand,3379  // since the index mask operand is the zeroinitializer (undef and3380  // poison are treated as zeroinitializer here).3381  if (U.getOperand(0)->getType()->isScalableTy()) {3382    Register Val = getOrCreateVReg(*U.getOperand(0));3383    auto SplatVal = MIRBuilder.buildExtractVectorElementConstant(3384        MRI->getType(Val).getElementType(), Val, 0);3385    MIRBuilder.buildSplatVector(getOrCreateVReg(U), SplatVal);3386    return true;3387  }3388 3389  ArrayRef<int> Mask;3390  if (auto *SVI = dyn_cast<ShuffleVectorInst>(&U))3391    Mask = SVI->getShuffleMask();3392  else3393    Mask = cast<ConstantExpr>(U).getShuffleMask();3394 3395  // As GISel does not represent <1 x > vectors as a separate type from scalars,3396  // we transform shuffle_vector with a scalar output to an3397  // ExtractVectorElement. If the input type is also scalar it becomes a Copy.3398  unsigned DstElts = cast<FixedVectorType>(U.getType())->getNumElements();3399  unsigned SrcElts =3400      cast<FixedVectorType>(U.getOperand(0)->getType())->getNumElements();3401  if (DstElts == 1) {3402    unsigned M = Mask[0];3403    if (SrcElts == 1) {3404      if (M == 0 || M == 1)3405        return translateCopy(U, *U.getOperand(M), MIRBuilder);3406      MIRBuilder.buildUndef(getOrCreateVReg(U));3407    } else {3408      Register Dst = getOrCreateVReg(U);3409      if (M < SrcElts) {3410        MIRBuilder.buildExtractVectorElementConstant(3411            Dst, getOrCreateVReg(*U.getOperand(0)), M);3412      } else if (M < SrcElts * 2) {3413        MIRBuilder.buildExtractVectorElementConstant(3414            Dst, getOrCreateVReg(*U.getOperand(1)), M - SrcElts);3415      } else {3416        MIRBuilder.buildUndef(Dst);3417      }3418    }3419    return true;3420  }3421 3422  // A single element src is transformed to a build_vector.3423  if (SrcElts == 1) {3424    SmallVector<Register> Ops;3425    Register Undef;3426    for (int M : Mask) {3427      LLT SrcTy = getLLTForType(*U.getOperand(0)->getType(), *DL);3428      if (M == 0 || M == 1) {3429        Ops.push_back(getOrCreateVReg(*U.getOperand(M)));3430      } else {3431        if (!Undef.isValid()) {3432          Undef = MRI->createGenericVirtualRegister(SrcTy);3433          MIRBuilder.buildUndef(Undef);3434        }3435        Ops.push_back(Undef);3436      }3437    }3438    MIRBuilder.buildBuildVector(getOrCreateVReg(U), Ops);3439    return true;3440  }3441 3442  ArrayRef<int> MaskAlloc = MF->allocateShuffleMask(Mask);3443  MIRBuilder3444      .buildInstr(TargetOpcode::G_SHUFFLE_VECTOR, {getOrCreateVReg(U)},3445                  {getOrCreateVReg(*U.getOperand(0)),3446                   getOrCreateVReg(*U.getOperand(1))})3447      .addShuffleMask(MaskAlloc);3448  return true;3449}3450 3451bool IRTranslator::translatePHI(const User &U, MachineIRBuilder &MIRBuilder) {3452  const PHINode &PI = cast<PHINode>(U);3453 3454  SmallVector<MachineInstr *, 4> Insts;3455  for (auto Reg : getOrCreateVRegs(PI)) {3456    auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_PHI, {Reg}, {});3457    Insts.push_back(MIB.getInstr());3458  }3459 3460  PendingPHIs.emplace_back(&PI, std::move(Insts));3461  return true;3462}3463 3464bool IRTranslator::translateAtomicCmpXchg(const User &U,3465                                          MachineIRBuilder &MIRBuilder) {3466  const AtomicCmpXchgInst &I = cast<AtomicCmpXchgInst>(U);3467 3468  auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);3469 3470  auto Res = getOrCreateVRegs(I);3471  Register OldValRes = Res[0];3472  Register SuccessRes = Res[1];3473  Register Addr = getOrCreateVReg(*I.getPointerOperand());3474  Register Cmp = getOrCreateVReg(*I.getCompareOperand());3475  Register NewVal = getOrCreateVReg(*I.getNewValOperand());3476 3477  MIRBuilder.buildAtomicCmpXchgWithSuccess(3478      OldValRes, SuccessRes, Addr, Cmp, NewVal,3479      *MF->getMachineMemOperand(3480          MachinePointerInfo(I.getPointerOperand()), Flags, MRI->getType(Cmp),3481          getMemOpAlign(I), I.getAAMetadata(), nullptr, I.getSyncScopeID(),3482          I.getSuccessOrdering(), I.getFailureOrdering()));3483  return true;3484}3485 3486bool IRTranslator::translateAtomicRMW(const User &U,3487                                      MachineIRBuilder &MIRBuilder) {3488  if (containsBF16Type(U) && !targetSupportsBF16Type(MF))3489    return false;3490 3491  const AtomicRMWInst &I = cast<AtomicRMWInst>(U);3492  auto Flags = TLI->getAtomicMemOperandFlags(I, *DL);3493 3494  Register Res = getOrCreateVReg(I);3495  Register Addr = getOrCreateVReg(*I.getPointerOperand());3496  Register Val = getOrCreateVReg(*I.getValOperand());3497 3498  unsigned Opcode = 0;3499  switch (I.getOperation()) {3500  default:3501    return false;3502  case AtomicRMWInst::Xchg:3503    Opcode = TargetOpcode::G_ATOMICRMW_XCHG;3504    break;3505  case AtomicRMWInst::Add:3506    Opcode = TargetOpcode::G_ATOMICRMW_ADD;3507    break;3508  case AtomicRMWInst::Sub:3509    Opcode = TargetOpcode::G_ATOMICRMW_SUB;3510    break;3511  case AtomicRMWInst::And:3512    Opcode = TargetOpcode::G_ATOMICRMW_AND;3513    break;3514  case AtomicRMWInst::Nand:3515    Opcode = TargetOpcode::G_ATOMICRMW_NAND;3516    break;3517  case AtomicRMWInst::Or:3518    Opcode = TargetOpcode::G_ATOMICRMW_OR;3519    break;3520  case AtomicRMWInst::Xor:3521    Opcode = TargetOpcode::G_ATOMICRMW_XOR;3522    break;3523  case AtomicRMWInst::Max:3524    Opcode = TargetOpcode::G_ATOMICRMW_MAX;3525    break;3526  case AtomicRMWInst::Min:3527    Opcode = TargetOpcode::G_ATOMICRMW_MIN;3528    break;3529  case AtomicRMWInst::UMax:3530    Opcode = TargetOpcode::G_ATOMICRMW_UMAX;3531    break;3532  case AtomicRMWInst::UMin:3533    Opcode = TargetOpcode::G_ATOMICRMW_UMIN;3534    break;3535  case AtomicRMWInst::FAdd:3536    Opcode = TargetOpcode::G_ATOMICRMW_FADD;3537    break;3538  case AtomicRMWInst::FSub:3539    Opcode = TargetOpcode::G_ATOMICRMW_FSUB;3540    break;3541  case AtomicRMWInst::FMax:3542    Opcode = TargetOpcode::G_ATOMICRMW_FMAX;3543    break;3544  case AtomicRMWInst::FMin:3545    Opcode = TargetOpcode::G_ATOMICRMW_FMIN;3546    break;3547  case AtomicRMWInst::FMaximum:3548    Opcode = TargetOpcode::G_ATOMICRMW_FMAXIMUM;3549    break;3550  case AtomicRMWInst::FMinimum:3551    Opcode = TargetOpcode::G_ATOMICRMW_FMINIMUM;3552    break;3553  case AtomicRMWInst::UIncWrap:3554    Opcode = TargetOpcode::G_ATOMICRMW_UINC_WRAP;3555    break;3556  case AtomicRMWInst::UDecWrap:3557    Opcode = TargetOpcode::G_ATOMICRMW_UDEC_WRAP;3558    break;3559  case AtomicRMWInst::USubCond:3560    Opcode = TargetOpcode::G_ATOMICRMW_USUB_COND;3561    break;3562  case AtomicRMWInst::USubSat:3563    Opcode = TargetOpcode::G_ATOMICRMW_USUB_SAT;3564    break;3565  }3566 3567  MIRBuilder.buildAtomicRMW(3568      Opcode, Res, Addr, Val,3569      *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),3570                                Flags, MRI->getType(Val), getMemOpAlign(I),3571                                I.getAAMetadata(), nullptr, I.getSyncScopeID(),3572                                I.getOrdering()));3573  return true;3574}3575 3576bool IRTranslator::translateFence(const User &U,3577                                  MachineIRBuilder &MIRBuilder) {3578  const FenceInst &Fence = cast<FenceInst>(U);3579  MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),3580                        Fence.getSyncScopeID());3581  return true;3582}3583 3584bool IRTranslator::translateFreeze(const User &U,3585                                   MachineIRBuilder &MIRBuilder) {3586  const ArrayRef<Register> DstRegs = getOrCreateVRegs(U);3587  const ArrayRef<Register> SrcRegs = getOrCreateVRegs(*U.getOperand(0));3588 3589  assert(DstRegs.size() == SrcRegs.size() &&3590         "Freeze with different source and destination type?");3591 3592  for (unsigned I = 0; I < DstRegs.size(); ++I) {3593    MIRBuilder.buildFreeze(DstRegs[I], SrcRegs[I]);3594  }3595 3596  return true;3597}3598 3599void IRTranslator::finishPendingPhis() {3600#ifndef NDEBUG3601  DILocationVerifier Verifier;3602  GISelObserverWrapper WrapperObserver(&Verifier);3603  RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);3604#endif // ifndef NDEBUG3605  for (auto &Phi : PendingPHIs) {3606    const PHINode *PI = Phi.first;3607    if (PI->getType()->isEmptyTy())3608      continue;3609    ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;3610    MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent();3611    EntryBuilder->setDebugLoc(PI->getDebugLoc());3612#ifndef NDEBUG3613    Verifier.setCurrentInst(PI);3614#endif // ifndef NDEBUG3615 3616    SmallPtrSet<const MachineBasicBlock *, 16> SeenPreds;3617    for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {3618      auto IRPred = PI->getIncomingBlock(i);3619      ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));3620      for (auto *Pred : getMachinePredBBs({IRPred, PI->getParent()})) {3621        if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred))3622          continue;3623        SeenPreds.insert(Pred);3624        for (unsigned j = 0; j < ValRegs.size(); ++j) {3625          MachineInstrBuilder MIB(*MF, ComponentPHIs[j]);3626          MIB.addUse(ValRegs[j]);3627          MIB.addMBB(Pred);3628        }3629      }3630    }3631  }3632}3633 3634void IRTranslator::translateDbgValueRecord(Value *V, bool HasArgList,3635                                     const DILocalVariable *Variable,3636                                     const DIExpression *Expression,3637                                     const DebugLoc &DL,3638                                     MachineIRBuilder &MIRBuilder) {3639  assert(Variable->isValidLocationForIntrinsic(DL) &&3640         "Expected inlined-at fields to agree");3641  // Act as if we're handling a debug intrinsic.3642  MIRBuilder.setDebugLoc(DL);3643 3644  if (!V || HasArgList) {3645    // DI cannot produce a valid DBG_VALUE, so produce an undef DBG_VALUE to3646    // terminate any prior location.3647    MIRBuilder.buildIndirectDbgValue(0, Variable, Expression);3648    return;3649  }3650 3651  if (const auto *CI = dyn_cast<Constant>(V)) {3652    MIRBuilder.buildConstDbgValue(*CI, Variable, Expression);3653    return;3654  }3655 3656  if (auto *AI = dyn_cast<AllocaInst>(V);3657      AI && AI->isStaticAlloca() && Expression->startsWithDeref()) {3658    // If the value is an alloca and the expression starts with a3659    // dereference, track a stack slot instead of a register, as registers3660    // may be clobbered.3661    auto ExprOperands = Expression->getElements();3662    auto *ExprDerefRemoved =3663        DIExpression::get(AI->getContext(), ExprOperands.drop_front());3664    MIRBuilder.buildFIDbgValue(getOrCreateFrameIndex(*AI), Variable,3665                               ExprDerefRemoved);3666    return;3667  }3668  if (translateIfEntryValueArgument(false, V, Variable, Expression, DL,3669                                    MIRBuilder))3670    return;3671  for (Register Reg : getOrCreateVRegs(*V)) {3672    // FIXME: This does not handle register-indirect values at offset 0. The3673    // direct/indirect thing shouldn't really be handled by something as3674    // implicit as reg+noreg vs reg+imm in the first place, but it seems3675    // pretty baked in right now.3676    MIRBuilder.buildDirectDbgValue(Reg, Variable, Expression);3677  }3678}3679 3680void IRTranslator::translateDbgDeclareRecord(Value *Address, bool HasArgList,3681                                     const DILocalVariable *Variable,3682                                     const DIExpression *Expression,3683                                     const DebugLoc &DL,3684                                     MachineIRBuilder &MIRBuilder) {3685  if (!Address || isa<UndefValue>(Address)) {3686    LLVM_DEBUG(dbgs() << "Dropping debug info for " << *Variable << "\n");3687    return;3688  }3689 3690  assert(Variable->isValidLocationForIntrinsic(DL) &&3691         "Expected inlined-at fields to agree");3692  auto AI = dyn_cast<AllocaInst>(Address);3693  if (AI && AI->isStaticAlloca()) {3694    // Static allocas are tracked at the MF level, no need for DBG_VALUE3695    // instructions (in fact, they get ignored if they *do* exist).3696    MF->setVariableDbgInfo(Variable, Expression,3697                           getOrCreateFrameIndex(*AI), DL);3698    return;3699  }3700 3701  if (translateIfEntryValueArgument(true, Address, Variable,3702                                    Expression, DL,3703                                    MIRBuilder))3704    return;3705 3706  // A dbg.declare describes the address of a source variable, so lower it3707  // into an indirect DBG_VALUE.3708  MIRBuilder.setDebugLoc(DL);3709  MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address), Variable,3710                                   Expression);3711}3712 3713void IRTranslator::translateDbgInfo(const Instruction &Inst,3714                                      MachineIRBuilder &MIRBuilder) {3715  for (DbgRecord &DR : Inst.getDbgRecordRange()) {3716    if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(&DR)) {3717      MIRBuilder.setDebugLoc(DLR->getDebugLoc());3718      assert(DLR->getLabel() && "Missing label");3719      assert(DLR->getLabel()->isValidLocationForIntrinsic(3720                 MIRBuilder.getDebugLoc()) &&3721             "Expected inlined-at fields to agree");3722      MIRBuilder.buildDbgLabel(DLR->getLabel());3723      continue;3724    }3725    DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);3726    const DILocalVariable *Variable = DVR.getVariable();3727    const DIExpression *Expression = DVR.getExpression();3728    Value *V = DVR.getVariableLocationOp(0);3729    if (DVR.isDbgDeclare())3730      translateDbgDeclareRecord(V, DVR.hasArgList(), Variable, Expression,3731                                DVR.getDebugLoc(), MIRBuilder);3732    else3733      translateDbgValueRecord(V, DVR.hasArgList(), Variable, Expression,3734                              DVR.getDebugLoc(), MIRBuilder);3735  }3736}3737 3738bool IRTranslator::translate(const Instruction &Inst) {3739  CurBuilder->setDebugLoc(Inst.getDebugLoc());3740  CurBuilder->setPCSections(Inst.getMetadata(LLVMContext::MD_pcsections));3741  CurBuilder->setMMRAMetadata(Inst.getMetadata(LLVMContext::MD_mmra));3742 3743  if (TLI->fallBackToDAGISel(Inst))3744    return false;3745 3746  switch (Inst.getOpcode()) {3747#define HANDLE_INST(NUM, OPCODE, CLASS)                                        \3748  case Instruction::OPCODE:                                                    \3749    return translate##OPCODE(Inst, *CurBuilder.get());3750#include "llvm/IR/Instruction.def"3751  default:3752    return false;3753  }3754}3755 3756bool IRTranslator::translate(const Constant &C, Register Reg) {3757  // We only emit constants into the entry block from here. To prevent jumpy3758  // debug behaviour remove debug line.3759  if (auto CurrInstDL = CurBuilder->getDL())3760    EntryBuilder->setDebugLoc(DebugLoc());3761 3762  if (auto CI = dyn_cast<ConstantInt>(&C)) {3763    // buildConstant expects a to-be-splatted scalar ConstantInt.3764    if (isa<VectorType>(CI->getType()))3765      CI = ConstantInt::get(CI->getContext(), CI->getValue());3766    EntryBuilder->buildConstant(Reg, *CI);3767  } else if (auto CF = dyn_cast<ConstantFP>(&C)) {3768    // buildFConstant expects a to-be-splatted scalar ConstantFP.3769    if (isa<VectorType>(CF->getType()))3770      CF = ConstantFP::get(CF->getContext(), CF->getValue());3771    EntryBuilder->buildFConstant(Reg, *CF);3772  } else if (isa<UndefValue>(C))3773    EntryBuilder->buildUndef(Reg);3774  else if (isa<ConstantPointerNull>(C))3775    EntryBuilder->buildConstant(Reg, 0);3776  else if (auto GV = dyn_cast<GlobalValue>(&C))3777    EntryBuilder->buildGlobalValue(Reg, GV);3778  else if (auto CPA = dyn_cast<ConstantPtrAuth>(&C)) {3779    Register Addr = getOrCreateVReg(*CPA->getPointer());3780    Register AddrDisc = getOrCreateVReg(*CPA->getAddrDiscriminator());3781    EntryBuilder->buildConstantPtrAuth(Reg, CPA, Addr, AddrDisc);3782  } else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {3783    Constant &Elt = *CAZ->getElementValue(0u);3784    if (isa<ScalableVectorType>(CAZ->getType())) {3785      EntryBuilder->buildSplatVector(Reg, getOrCreateVReg(Elt));3786      return true;3787    }3788    // Return the scalar if it is a <1 x Ty> vector.3789    unsigned NumElts = CAZ->getElementCount().getFixedValue();3790    if (NumElts == 1)3791      return translateCopy(C, Elt, *EntryBuilder);3792    // All elements are zero so we can just use the first one.3793    EntryBuilder->buildSplatBuildVector(Reg, getOrCreateVReg(Elt));3794  } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {3795    // Return the scalar if it is a <1 x Ty> vector.3796    if (CV->getNumElements() == 1)3797      return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder);3798    SmallVector<Register, 4> Ops;3799    for (unsigned i = 0; i < CV->getNumElements(); ++i) {3800      Constant &Elt = *CV->getElementAsConstant(i);3801      Ops.push_back(getOrCreateVReg(Elt));3802    }3803    EntryBuilder->buildBuildVector(Reg, Ops);3804  } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {3805    switch(CE->getOpcode()) {3806#define HANDLE_INST(NUM, OPCODE, CLASS)                                        \3807  case Instruction::OPCODE:                                                    \3808    return translate##OPCODE(*CE, *EntryBuilder.get());3809#include "llvm/IR/Instruction.def"3810    default:3811      return false;3812    }3813  } else if (auto CV = dyn_cast<ConstantVector>(&C)) {3814    if (CV->getNumOperands() == 1)3815      return translateCopy(C, *CV->getOperand(0), *EntryBuilder);3816    SmallVector<Register, 4> Ops;3817    for (unsigned i = 0; i < CV->getNumOperands(); ++i) {3818      Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));3819    }3820    EntryBuilder->buildBuildVector(Reg, Ops);3821  } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {3822    EntryBuilder->buildBlockAddress(Reg, BA);3823  } else3824    return false;3825 3826  return true;3827}3828 3829bool IRTranslator::finalizeBasicBlock(const BasicBlock &BB,3830                                      MachineBasicBlock &MBB) {3831  for (auto &BTB : SL->BitTestCases) {3832    // Emit header first, if it wasn't already emitted.3833    if (!BTB.Emitted)3834      emitBitTestHeader(BTB, BTB.Parent);3835 3836    BranchProbability UnhandledProb = BTB.Prob;3837    for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {3838      UnhandledProb -= BTB.Cases[j].ExtraProb;3839      // Set the current basic block to the mbb we wish to insert the code into3840      MachineBasicBlock *MBB = BTB.Cases[j].ThisBB;3841      // If all cases cover a contiguous range, it is not necessary to jump to3842      // the default block after the last bit test fails. This is because the3843      // range check during bit test header creation has guaranteed that every3844      // case here doesn't go outside the range. In this case, there is no need3845      // to perform the last bit test, as it will always be true. Instead, make3846      // the second-to-last bit-test fall through to the target of the last bit3847      // test, and delete the last bit test.3848 3849      MachineBasicBlock *NextMBB;3850      if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {3851        // Second-to-last bit-test with contiguous range: fall through to the3852        // target of the final bit test.3853        NextMBB = BTB.Cases[j + 1].TargetBB;3854      } else if (j + 1 == ej) {3855        // For the last bit test, fall through to Default.3856        NextMBB = BTB.Default;3857      } else {3858        // Otherwise, fall through to the next bit test.3859        NextMBB = BTB.Cases[j + 1].ThisBB;3860      }3861 3862      emitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j], MBB);3863 3864      if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {3865        // We need to record the replacement phi edge here that normally3866        // happens in emitBitTestCase before we delete the case, otherwise the3867        // phi edge will be lost.3868        addMachineCFGPred({BTB.Parent->getBasicBlock(),3869                           BTB.Cases[ej - 1].TargetBB->getBasicBlock()},3870                          MBB);3871        // Since we're not going to use the final bit test, remove it.3872        BTB.Cases.pop_back();3873        break;3874      }3875    }3876    // This is "default" BB. We have two jumps to it. From "header" BB and from3877    // last "case" BB, unless the latter was skipped.3878    CFGEdge HeaderToDefaultEdge = {BTB.Parent->getBasicBlock(),3879                                   BTB.Default->getBasicBlock()};3880    addMachineCFGPred(HeaderToDefaultEdge, BTB.Parent);3881    if (!BTB.ContiguousRange) {3882      addMachineCFGPred(HeaderToDefaultEdge, BTB.Cases.back().ThisBB);3883    }3884  }3885  SL->BitTestCases.clear();3886 3887  for (auto &JTCase : SL->JTCases) {3888    // Emit header first, if it wasn't already emitted.3889    if (!JTCase.first.Emitted)3890      emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB);3891 3892    emitJumpTable(JTCase.second, JTCase.second.MBB);3893  }3894  SL->JTCases.clear();3895 3896  for (auto &SwCase : SL->SwitchCases)3897    emitSwitchCase(SwCase, &CurBuilder->getMBB(), *CurBuilder);3898  SL->SwitchCases.clear();3899 3900  // Check if we need to generate stack-protector guard checks.3901  StackProtector &SP = getAnalysis<StackProtector>();3902  if (SP.shouldEmitSDCheck(BB)) {3903    bool FunctionBasedInstrumentation =3904        TLI->getSSPStackGuardCheck(*MF->getFunction().getParent());3905    SPDescriptor.initialize(&BB, &MBB, FunctionBasedInstrumentation);3906  }3907  // Handle stack protector.3908  if (SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {3909    LLVM_DEBUG(dbgs() << "Unimplemented stack protector case\n");3910    return false;3911  } else if (SPDescriptor.shouldEmitStackProtector()) {3912    MachineBasicBlock *ParentMBB = SPDescriptor.getParentMBB();3913    MachineBasicBlock *SuccessMBB = SPDescriptor.getSuccessMBB();3914 3915    // Find the split point to split the parent mbb. At the same time copy all3916    // physical registers used in the tail of parent mbb into virtual registers3917    // before the split point and back into physical registers after the split3918    // point. This prevents us needing to deal with Live-ins and many other3919    // register allocation issues caused by us splitting the parent mbb. The3920    // register allocator will clean up said virtual copies later on.3921    MachineBasicBlock::iterator SplitPoint = findSplitPointForStackProtector(3922        ParentMBB, *MF->getSubtarget().getInstrInfo());3923 3924    // Splice the terminator of ParentMBB into SuccessMBB.3925    SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,3926                       ParentMBB->end());3927 3928    // Add compare/jump on neq/jump to the parent BB.3929    if (!emitSPDescriptorParent(SPDescriptor, ParentMBB))3930      return false;3931 3932    // CodeGen Failure MBB if we have not codegened it yet.3933    MachineBasicBlock *FailureMBB = SPDescriptor.getFailureMBB();3934    if (FailureMBB->empty()) {3935      if (!emitSPDescriptorFailure(SPDescriptor, FailureMBB))3936        return false;3937    }3938 3939    // Clear the Per-BB State.3940    SPDescriptor.resetPerBBState();3941  }3942  return true;3943}3944 3945bool IRTranslator::emitSPDescriptorParent(StackProtectorDescriptor &SPD,3946                                          MachineBasicBlock *ParentBB) {3947  CurBuilder->setInsertPt(*ParentBB, ParentBB->end());3948  // First create the loads to the guard/stack slot for the comparison.3949  Type *PtrIRTy = PointerType::getUnqual(MF->getFunction().getContext());3950  const LLT PtrTy = getLLTForType(*PtrIRTy, *DL);3951  LLT PtrMemTy = getLLTForMVT(TLI->getPointerMemTy(*DL));3952 3953  MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();3954  int FI = MFI.getStackProtectorIndex();3955 3956  Register Guard;3957  Register StackSlotPtr = CurBuilder->buildFrameIndex(PtrTy, FI).getReg(0);3958  const Module &M = *ParentBB->getParent()->getFunction().getParent();3959  Align Align = DL->getPrefTypeAlign(PointerType::getUnqual(M.getContext()));3960 3961  // Generate code to load the content of the guard slot.3962  Register GuardVal =3963      CurBuilder3964          ->buildLoad(PtrMemTy, StackSlotPtr,3965                      MachinePointerInfo::getFixedStack(*MF, FI), Align,3966                      MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile)3967          .getReg(0);3968 3969  if (TLI->useStackGuardXorFP()) {3970    LLVM_DEBUG(dbgs() << "Stack protector xor'ing with FP not yet implemented");3971    return false;3972  }3973 3974  // Retrieve guard check function, nullptr if instrumentation is inlined.3975  if (const Function *GuardCheckFn = TLI->getSSPStackGuardCheck(M)) {3976    // This path is currently untestable on GlobalISel, since the only platform3977    // that needs this seems to be Windows, and we fall back on that currently.3978    // The code still lives here in case that changes.3979    // Silence warning about unused variable until the code below that uses3980    // 'GuardCheckFn' is enabled.3981    (void)GuardCheckFn;3982    return false;3983#if 03984    // The target provides a guard check function to validate the guard value.3985    // Generate a call to that function with the content of the guard slot as3986    // argument.3987    FunctionType *FnTy = GuardCheckFn->getFunctionType();3988    assert(FnTy->getNumParams() == 1 && "Invalid function signature");3989    ISD::ArgFlagsTy Flags;3990    if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))3991      Flags.setInReg();3992    CallLowering::ArgInfo GuardArgInfo(3993        {GuardVal, FnTy->getParamType(0), {Flags}});3994 3995    CallLowering::CallLoweringInfo Info;3996    Info.OrigArgs.push_back(GuardArgInfo);3997    Info.CallConv = GuardCheckFn->getCallingConv();3998    Info.Callee = MachineOperand::CreateGA(GuardCheckFn, 0);3999    Info.OrigRet = {Register(), FnTy->getReturnType()};4000    if (!CLI->lowerCall(MIRBuilder, Info)) {4001      LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector check\n");4002      return false;4003    }4004    return true;4005#endif4006  }4007 4008  // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.4009  // Otherwise, emit a volatile load to retrieve the stack guard value.4010  if (TLI->useLoadStackGuardNode(*ParentBB->getBasicBlock()->getModule())) {4011    Guard =4012        MRI->createGenericVirtualRegister(LLT::scalar(PtrTy.getSizeInBits()));4013    getStackGuard(Guard, *CurBuilder);4014  } else {4015    // TODO: test using android subtarget when we support @llvm.thread.pointer.4016    const Value *IRGuard = TLI->getSDagStackGuard(M);4017    Register GuardPtr = getOrCreateVReg(*IRGuard);4018 4019    Guard = CurBuilder4020                ->buildLoad(PtrMemTy, GuardPtr,4021                            MachinePointerInfo::getFixedStack(*MF, FI), Align,4022                            MachineMemOperand::MOLoad |4023                                MachineMemOperand::MOVolatile)4024                .getReg(0);4025  }4026 4027  // Perform the comparison.4028  auto Cmp =4029      CurBuilder->buildICmp(CmpInst::ICMP_NE, LLT::scalar(1), Guard, GuardVal);4030  // If the guard/stackslot do not equal, branch to failure MBB.4031  CurBuilder->buildBrCond(Cmp, *SPD.getFailureMBB());4032  // Otherwise branch to success MBB.4033  CurBuilder->buildBr(*SPD.getSuccessMBB());4034  return true;4035}4036 4037bool IRTranslator::emitSPDescriptorFailure(StackProtectorDescriptor &SPD,4038                                           MachineBasicBlock *FailureBB) {4039  CurBuilder->setInsertPt(*FailureBB, FailureBB->end());4040 4041  const RTLIB::Libcall Libcall = RTLIB::STACKPROTECTOR_CHECK_FAIL;4042  const char *Name = TLI->getLibcallName(Libcall);4043 4044  CallLowering::CallLoweringInfo Info;4045  Info.CallConv = TLI->getLibcallCallingConv(Libcall);4046  Info.Callee = MachineOperand::CreateES(Name);4047  Info.OrigRet = {Register(), Type::getVoidTy(MF->getFunction().getContext()),4048                  0};4049  if (!CLI->lowerCall(*CurBuilder, Info)) {4050    LLVM_DEBUG(dbgs() << "Failed to lower call to stack protector fail\n");4051    return false;4052  }4053 4054  // Emit a trap instruction if we are required to do so.4055  const TargetOptions &TargetOpts = TLI->getTargetMachine().Options;4056  if (TargetOpts.TrapUnreachable && !TargetOpts.NoTrapAfterNoreturn)4057    CurBuilder->buildInstr(TargetOpcode::G_TRAP);4058 4059  return true;4060}4061 4062void IRTranslator::finalizeFunction() {4063  // Release the memory used by the different maps we4064  // needed during the translation.4065  PendingPHIs.clear();4066  VMap.reset();4067  FrameIndices.clear();4068  MachinePreds.clear();4069  // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it4070  // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid4071  // destroying it twice (in ~IRTranslator() and ~LLVMContext())4072  EntryBuilder.reset();4073  CurBuilder.reset();4074  FuncInfo.clear();4075  SPDescriptor.resetPerFunctionState();4076}4077 4078/// Returns true if a BasicBlock \p BB within a variadic function contains a4079/// variadic musttail call.4080static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {4081  if (!IsVarArg)4082    return false;4083 4084  // Walk the block backwards, because tail calls usually only appear at the end4085  // of a block.4086  return llvm::any_of(llvm::reverse(BB), [](const Instruction &I) {4087    const auto *CI = dyn_cast<CallInst>(&I);4088    return CI && CI->isMustTailCall();4089  });4090}4091 4092bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {4093  MF = &CurMF;4094  const Function &F = MF->getFunction();4095  GISelCSEAnalysisWrapper &Wrapper =4096      getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();4097  // Set the CSEConfig and run the analysis.4098  GISelCSEInfo *CSEInfo = nullptr;4099  TPC = &getAnalysis<TargetPassConfig>();4100  bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()4101                       ? EnableCSEInIRTranslator4102                       : TPC->isGISelCSEEnabled();4103  TLI = MF->getSubtarget().getTargetLowering();4104 4105  if (EnableCSE) {4106    EntryBuilder = std::make_unique<CSEMIRBuilder>(CurMF);4107    CSEInfo = &Wrapper.get(TPC->getCSEConfig());4108    EntryBuilder->setCSEInfo(CSEInfo);4109    CurBuilder = std::make_unique<CSEMIRBuilder>(CurMF);4110    CurBuilder->setCSEInfo(CSEInfo);4111  } else {4112    EntryBuilder = std::make_unique<MachineIRBuilder>();4113    CurBuilder = std::make_unique<MachineIRBuilder>();4114  }4115  CLI = MF->getSubtarget().getCallLowering();4116  CurBuilder->setMF(*MF);4117  EntryBuilder->setMF(*MF);4118  MRI = &MF->getRegInfo();4119  DL = &F.getDataLayout();4120  ORE = std::make_unique<OptimizationRemarkEmitter>(&F);4121  const TargetMachine &TM = MF->getTarget();4122  TM.resetTargetOptions(F);4123  EnableOpts = OptLevel != CodeGenOptLevel::None && !skipFunction(F);4124  FuncInfo.MF = MF;4125  if (EnableOpts) {4126    AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();4127    FuncInfo.BPI = &getAnalysis<BranchProbabilityInfoWrapperPass>().getBPI();4128  } else {4129    AA = nullptr;4130    FuncInfo.BPI = nullptr;4131  }4132 4133  AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(4134      MF->getFunction());4135  LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);4136  FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);4137 4138  SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);4139  SL->init(*TLI, TM, *DL);4140 4141  assert(PendingPHIs.empty() && "stale PHIs");4142 4143  // Targets which want to use big endian can enable it using4144  // enableBigEndian()4145  if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {4146    // Currently we don't properly handle big endian code.4147    OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",4148                               F.getSubprogram(), &F.getEntryBlock());4149    R << "unable to translate in big endian mode";4150    reportTranslationError(*MF, *TPC, *ORE, R);4151    return false;4152  }4153 4154  // Release the per-function state when we return, whether we succeeded or not.4155  auto FinalizeOnReturn = make_scope_exit([this]() { finalizeFunction(); });4156 4157  // Setup a separate basic-block for the arguments and constants4158  MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();4159  MF->push_back(EntryBB);4160  EntryBuilder->setMBB(*EntryBB);4161 4162  DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHIIt()->getDebugLoc();4163  SwiftError.setFunction(CurMF);4164  SwiftError.createEntriesInEntryBlock(DbgLoc);4165 4166  bool IsVarArg = F.isVarArg();4167  bool HasMustTailInVarArgFn = false;4168 4169  // Create all blocks, in IR order, to preserve the layout.4170  FuncInfo.MBBMap.resize(F.getMaxBlockNumber());4171  for (const BasicBlock &BB: F) {4172    auto *&MBB = FuncInfo.MBBMap[BB.getNumber()];4173 4174    MBB = MF->CreateMachineBasicBlock(&BB);4175    MF->push_back(MBB);4176 4177    if (BB.hasAddressTaken())4178      MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));4179 4180    if (!HasMustTailInVarArgFn)4181      HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);4182  }4183 4184  MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);4185 4186  // Make our arguments/constants entry block fallthrough to the IR entry block.4187  EntryBB->addSuccessor(&getMBB(F.front()));4188 4189  if (CLI->fallBackToDAGISel(*MF)) {4190    OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",4191                               F.getSubprogram(), &F.getEntryBlock());4192    R << "unable to lower function: "4193      << ore::NV("Prototype", F.getFunctionType());4194    reportTranslationError(*MF, *TPC, *ORE, R);4195    return false;4196  }4197 4198  // Lower the actual args into this basic block.4199  SmallVector<ArrayRef<Register>, 8> VRegArgs;4200  for (const Argument &Arg: F.args()) {4201    if (DL->getTypeStoreSize(Arg.getType()).isZero())4202      continue; // Don't handle zero sized types.4203    ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);4204    VRegArgs.push_back(VRegs);4205 4206    if (Arg.hasSwiftErrorAttr()) {4207      assert(VRegs.size() == 1 && "Too many vregs for Swift error");4208      SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);4209    }4210  }4211 4212  if (!CLI->lowerFormalArguments(*EntryBuilder, F, VRegArgs, FuncInfo)) {4213    OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",4214                               F.getSubprogram(), &F.getEntryBlock());4215    R << "unable to lower arguments: "4216      << ore::NV("Prototype", F.getFunctionType());4217    reportTranslationError(*MF, *TPC, *ORE, R);4218    return false;4219  }4220 4221  // Need to visit defs before uses when translating instructions.4222  GISelObserverWrapper WrapperObserver;4223  if (EnableCSE && CSEInfo)4224    WrapperObserver.addObserver(CSEInfo);4225  {4226    ReversePostOrderTraversal<const Function *> RPOT(&F);4227#ifndef NDEBUG4228    DILocationVerifier Verifier;4229    WrapperObserver.addObserver(&Verifier);4230#endif // ifndef NDEBUG4231    RAIIMFObsDelInstaller ObsInstall(*MF, WrapperObserver);4232    for (const BasicBlock *BB : RPOT) {4233      MachineBasicBlock &MBB = getMBB(*BB);4234      // Set the insertion point of all the following translations to4235      // the end of this basic block.4236      CurBuilder->setMBB(MBB);4237      HasTailCall = false;4238      for (const Instruction &Inst : *BB) {4239        // If we translated a tail call in the last step, then we know4240        // everything after the call is either a return, or something that is4241        // handled by the call itself. (E.g. a lifetime marker or assume4242        // intrinsic.) In this case, we should stop translating the block and4243        // move on.4244        if (HasTailCall)4245          break;4246#ifndef NDEBUG4247        Verifier.setCurrentInst(&Inst);4248#endif // ifndef NDEBUG4249 4250        // Translate any debug-info attached to the instruction.4251        translateDbgInfo(Inst, *CurBuilder);4252 4253        if (translate(Inst))4254          continue;4255 4256        OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",4257                                   Inst.getDebugLoc(), BB);4258        R << "unable to translate instruction: " << ore::NV("Opcode", &Inst);4259 4260        if (ORE->allowExtraAnalysis("gisel-irtranslator")) {4261          std::string InstStrStorage;4262          raw_string_ostream InstStr(InstStrStorage);4263          InstStr << Inst;4264 4265          R << ": '" << InstStrStorage << "'";4266        }4267 4268        reportTranslationError(*MF, *TPC, *ORE, R);4269        return false;4270      }4271 4272      if (!finalizeBasicBlock(*BB, MBB)) {4273        OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",4274                                   BB->getTerminator()->getDebugLoc(), BB);4275        R << "unable to translate basic block";4276        reportTranslationError(*MF, *TPC, *ORE, R);4277        return false;4278      }4279    }4280#ifndef NDEBUG4281    WrapperObserver.removeObserver(&Verifier);4282#endif4283  }4284 4285  finishPendingPhis();4286 4287  SwiftError.propagateVRegs();4288 4289  // Merge the argument lowering and constants block with its single4290  // successor, the LLVM-IR entry block.  We want the basic block to4291  // be maximal.4292  assert(EntryBB->succ_size() == 1 &&4293         "Custom BB used for lowering should have only one successor");4294  // Get the successor of the current entry block.4295  MachineBasicBlock &NewEntryBB = **EntryBB->succ_begin();4296  assert(NewEntryBB.pred_size() == 1 &&4297         "LLVM-IR entry block has a predecessor!?");4298  // Move all the instruction from the current entry block to the4299  // new entry block.4300  NewEntryBB.splice(NewEntryBB.begin(), EntryBB, EntryBB->begin(),4301                    EntryBB->end());4302 4303  // Update the live-in information for the new entry block.4304  for (const MachineBasicBlock::RegisterMaskPair &LiveIn : EntryBB->liveins())4305    NewEntryBB.addLiveIn(LiveIn);4306  NewEntryBB.sortUniqueLiveIns();4307 4308  // Get rid of the now empty basic block.4309  EntryBB->removeSuccessor(&NewEntryBB);4310  MF->remove(EntryBB);4311  MF->deleteMachineBasicBlock(EntryBB);4312 4313  assert(&MF->front() == &NewEntryBB &&4314         "New entry wasn't next in the list of basic block!");4315 4316  // Initialize stack protector information.4317  StackProtector &SP = getAnalysis<StackProtector>();4318  SP.copyToMachineFrameInfo(MF->getFrameInfo());4319 4320  return false;4321}4322