brintos / llvm-project-archived public Read only

1//===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains code to lower X86 MachineInstrs to their corresponding10// MCInst records.11//12//===----------------------------------------------------------------------===//13 14#include "MCTargetDesc/X86ATTInstPrinter.h"15#include "MCTargetDesc/X86BaseInfo.h"16#include "MCTargetDesc/X86EncodingOptimization.h"17#include "MCTargetDesc/X86InstComments.h"18#include "MCTargetDesc/X86MCAsmInfo.h"19#include "MCTargetDesc/X86ShuffleDecode.h"20#include "MCTargetDesc/X86TargetStreamer.h"21#include "X86AsmPrinter.h"22#include "X86MachineFunctionInfo.h"23#include "X86RegisterInfo.h"24#include "X86ShuffleDecodeConstantPool.h"25#include "X86Subtarget.h"26#include "llvm/ADT/STLExtras.h"27#include "llvm/ADT/SmallString.h"28#include "llvm/ADT/StringExtras.h"29#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"30#include "llvm/CodeGen/MachineConstantPool.h"31#include "llvm/CodeGen/MachineFunction.h"32#include "llvm/CodeGen/MachineModuleInfoImpls.h"33#include "llvm/CodeGen/MachineOperand.h"34#include "llvm/CodeGen/StackMaps.h"35#include "llvm/CodeGen/WinEHFuncInfo.h"36#include "llvm/IR/DataLayout.h"37#include "llvm/IR/GlobalValue.h"38#include "llvm/IR/Mangler.h"39#include "llvm/MC/MCAsmInfo.h"40#include "llvm/MC/MCCodeEmitter.h"41#include "llvm/MC/MCContext.h"42#include "llvm/MC/MCExpr.h"43#include "llvm/MC/MCFixup.h"44#include "llvm/MC/MCInst.h"45#include "llvm/MC/MCInstBuilder.h"46#include "llvm/MC/MCSection.h"47#include "llvm/MC/MCStreamer.h"48#include "llvm/MC/MCSymbol.h"49#include "llvm/MC/TargetRegistry.h"50#include "llvm/Target/TargetLoweringObjectFile.h"51#include "llvm/Target/TargetMachine.h"52#include "llvm/Transforms/CFGuard.h"53#include "llvm/Transforms/Instrumentation/AddressSanitizer.h"54#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"55#include <string>56 57using namespace llvm;58 59static cl::opt<bool> EnableBranchHint("enable-branch-hint",60                                      cl::desc("Enable branch hint."),61                                      cl::init(false), cl::Hidden);62static cl::opt<unsigned> BranchHintProbabilityThreshold(63    "branch-hint-probability-threshold",64    cl::desc("The probability threshold of enabling branch hint."),65    cl::init(50), cl::Hidden);66 67namespace {68 69/// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.70class X86MCInstLower {71  MCContext &Ctx;72  const MachineFunction &MF;73  const TargetMachine &TM;74  const MCAsmInfo &MAI;75  X86AsmPrinter &AsmPrinter;76 77public:78  X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter);79 80  MCOperand LowerMachineOperand(const MachineInstr *MI,81                                const MachineOperand &MO) const;82  void Lower(const MachineInstr *MI, MCInst &OutMI) const;83 84  MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;85  MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;86 87private:88  MachineModuleInfoMachO &getMachOMMI() const;89};90 91} // end anonymous namespace92 93/// A RAII helper which defines a region of instructions which can't have94/// padding added between them for correctness.95struct NoAutoPaddingScope {96  MCStreamer &OS;97  const bool OldAllowAutoPadding;98  NoAutoPaddingScope(MCStreamer &OS)99      : OS(OS), OldAllowAutoPadding(OS.getAllowAutoPadding()) {100    changeAndComment(false);101  }102  ~NoAutoPaddingScope() { changeAndComment(OldAllowAutoPadding); }103  void changeAndComment(bool b) {104    if (b == OS.getAllowAutoPadding())105      return;106    OS.setAllowAutoPadding(b);107    if (b)108      OS.emitRawComment("autopadding");109    else110      OS.emitRawComment("noautopadding");111  }112};113 114// Emit a minimal sequence of nops spanning NumBytes bytes.115static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,116                        const X86Subtarget *Subtarget);117 118void X86AsmPrinter::StackMapShadowTracker::count(const MCInst &Inst,119                                                 const MCSubtargetInfo &STI,120                                                 MCCodeEmitter *CodeEmitter) {121  if (InShadow) {122    SmallString<256> Code;123    SmallVector<MCFixup, 4> Fixups;124    CodeEmitter->encodeInstruction(Inst, Code, Fixups, STI);125    CurrentShadowSize += Code.size();126    if (CurrentShadowSize >= RequiredShadowSize)127      InShadow = false; // The shadow is big enough. Stop counting.128  }129}130 131void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding(132    MCStreamer &OutStreamer, const MCSubtargetInfo &STI) {133  if (InShadow && CurrentShadowSize < RequiredShadowSize) {134    InShadow = false;135    emitX86Nops(OutStreamer, RequiredShadowSize - CurrentShadowSize,136                &MF->getSubtarget<X86Subtarget>());137  }138}139 140void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {141  OutStreamer->emitInstruction(Inst, getSubtargetInfo());142  SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get());143}144 145X86MCInstLower::X86MCInstLower(const MachineFunction &mf,146                               X86AsmPrinter &asmprinter)147    : Ctx(asmprinter.OutContext), MF(mf), TM(mf.getTarget()),148      MAI(*TM.getMCAsmInfo()), AsmPrinter(asmprinter) {}149 150MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {151  return AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoMachO>();152}153 154/// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol155/// operand to an MCSymbol.156MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const {157  const Triple &TT = TM.getTargetTriple();158  if (MO.isGlobal() && TT.isOSBinFormatELF())159    return AsmPrinter.getSymbolPreferLocal(*MO.getGlobal());160 161  const DataLayout &DL = MF.getDataLayout();162  assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) &&163         "Isn't a symbol reference");164 165  MCSymbol *Sym = nullptr;166  SmallString<128> Name;167  StringRef Suffix;168 169  switch (MO.getTargetFlags()) {170  case X86II::MO_DLLIMPORT:171    // Handle dllimport linkage.172    Name += "__imp_";173    break;174  case X86II::MO_COFFSTUB:175    Name += ".refptr.";176    break;177  case X86II::MO_DARWIN_NONLAZY:178  case X86II::MO_DARWIN_NONLAZY_PIC_BASE:179    Suffix = "$non_lazy_ptr";180    break;181  }182 183  if (!Suffix.empty())184    Name += DL.getPrivateGlobalPrefix();185 186  if (MO.isGlobal()) {187    const GlobalValue *GV = MO.getGlobal();188    AsmPrinter.getNameWithPrefix(Name, GV);189  } else if (MO.isSymbol()) {190    Mangler::getNameWithPrefix(Name, MO.getSymbolName(), DL);191  } else if (MO.isMBB()) {192    assert(Suffix.empty());193    Sym = MO.getMBB()->getSymbol();194  }195 196  Name += Suffix;197  if (!Sym)198    Sym = Ctx.getOrCreateSymbol(Name);199 200  // If the target flags on the operand changes the name of the symbol, do that201  // before we return the symbol.202  switch (MO.getTargetFlags()) {203  default:204    break;205  case X86II::MO_COFFSTUB: {206    MachineModuleInfoCOFF &MMICOFF =207        AsmPrinter.MMI->getObjFileInfo<MachineModuleInfoCOFF>();208    MachineModuleInfoImpl::StubValueTy &StubSym = MMICOFF.getGVStubEntry(Sym);209    if (!StubSym.getPointer()) {210      assert(MO.isGlobal() && "Extern symbol not handled yet");211      StubSym = MachineModuleInfoImpl::StubValueTy(212          AsmPrinter.getSymbol(MO.getGlobal()), true);213    }214    break;215  }216  case X86II::MO_DARWIN_NONLAZY:217  case X86II::MO_DARWIN_NONLAZY_PIC_BASE: {218    MachineModuleInfoImpl::StubValueTy &StubSym =219        getMachOMMI().getGVStubEntry(Sym);220    if (!StubSym.getPointer()) {221      assert(MO.isGlobal() && "Extern symbol not handled yet");222      StubSym = MachineModuleInfoImpl::StubValueTy(223          AsmPrinter.getSymbol(MO.getGlobal()),224          !MO.getGlobal()->hasInternalLinkage());225    }226    break;227  }228  }229 230  return Sym;231}232 233MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,234                                             MCSymbol *Sym) const {235  // FIXME: We would like an efficient form for this, so we don't have to do a236  // lot of extra uniquing.237  const MCExpr *Expr = nullptr;238  uint16_t Specifier = X86::S_None;239 240  switch (MO.getTargetFlags()) {241  default:242    llvm_unreachable("Unknown target flag on GV operand");243  case X86II::MO_NO_FLAG: // No flag.244  // These affect the name of the symbol, not any suffix.245  case X86II::MO_DARWIN_NONLAZY:246  case X86II::MO_DLLIMPORT:247  case X86II::MO_COFFSTUB:248    break;249 250  case X86II::MO_TLVP:251    Specifier = X86::S_TLVP;252    break;253  case X86II::MO_TLVP_PIC_BASE:254    Expr = MCSymbolRefExpr::create(Sym, X86::S_TLVP, Ctx);255    // Subtract the pic base.256    Expr = MCBinaryExpr::createSub(257        Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);258    break;259  case X86II::MO_SECREL:260    Specifier = uint16_t(X86::S_COFF_SECREL);261    break;262  case X86II::MO_TLSGD:263    Specifier = X86::S_TLSGD;264    break;265  case X86II::MO_TLSLD:266    Specifier = X86::S_TLSLD;267    break;268  case X86II::MO_TLSLDM:269    Specifier = X86::S_TLSLDM;270    break;271  case X86II::MO_GOTTPOFF:272    Specifier = X86::S_GOTTPOFF;273    break;274  case X86II::MO_INDNTPOFF:275    Specifier = X86::S_INDNTPOFF;276    break;277  case X86II::MO_TPOFF:278    Specifier = X86::S_TPOFF;279    break;280  case X86II::MO_DTPOFF:281    Specifier = X86::S_DTPOFF;282    break;283  case X86II::MO_NTPOFF:284    Specifier = X86::S_NTPOFF;285    break;286  case X86II::MO_GOTNTPOFF:287    Specifier = X86::S_GOTNTPOFF;288    break;289  case X86II::MO_GOTPCREL:290    Specifier = X86::S_GOTPCREL;291    break;292  case X86II::MO_GOTPCREL_NORELAX:293    Specifier = X86::S_GOTPCREL_NORELAX;294    break;295  case X86II::MO_GOT:296    Specifier = X86::S_GOT;297    break;298  case X86II::MO_GOTOFF:299    Specifier = X86::S_GOTOFF;300    break;301  case X86II::MO_PLT:302    Specifier = X86::S_PLT;303    break;304  case X86II::MO_ABS8:305    Specifier = X86::S_ABS8;306    break;307  case X86II::MO_PIC_BASE_OFFSET:308  case X86II::MO_DARWIN_NONLAZY_PIC_BASE:309    Expr = MCSymbolRefExpr::create(Sym, Ctx);310    // Subtract the pic base.311    Expr = MCBinaryExpr::createSub(312        Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);313    if (MO.isJTI()) {314      assert(MAI.doesSetDirectiveSuppressReloc());315      // If .set directive is supported, use it to reduce the number of316      // relocations the assembler will generate for differences between317      // local labels. This is only safe when the symbols are in the same318      // section so we are restricting it to jumptable references.319      MCSymbol *Label = Ctx.createTempSymbol();320      AsmPrinter.OutStreamer->emitAssignment(Label, Expr);321      Expr = MCSymbolRefExpr::create(Label, Ctx);322    }323    break;324  }325 326  if (!Expr)327    Expr = MCSymbolRefExpr::create(Sym, Specifier, Ctx);328 329  if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())330    Expr = MCBinaryExpr::createAdd(331        Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);332  return MCOperand::createExpr(Expr);333}334 335static unsigned getRetOpcode(const X86Subtarget &Subtarget) {336  return Subtarget.is64Bit() ? X86::RET64 : X86::RET32;337}338 339MCOperand X86MCInstLower::LowerMachineOperand(const MachineInstr *MI,340                                              const MachineOperand &MO) const {341  switch (MO.getType()) {342  default:343    MI->print(errs());344    llvm_unreachable("unknown operand type");345  case MachineOperand::MO_Register:346    // Ignore all implicit register operands.347    if (MO.isImplicit())348      return MCOperand();349    return MCOperand::createReg(MO.getReg());350  case MachineOperand::MO_Immediate:351    return MCOperand::createImm(MO.getImm());352  case MachineOperand::MO_MachineBasicBlock:353  case MachineOperand::MO_GlobalAddress:354  case MachineOperand::MO_ExternalSymbol:355    return LowerSymbolOperand(MO, GetSymbolFromOperand(MO));356  case MachineOperand::MO_MCSymbol:357    return LowerSymbolOperand(MO, MO.getMCSymbol());358  case MachineOperand::MO_JumpTableIndex:359    return LowerSymbolOperand(MO, AsmPrinter.GetJTISymbol(MO.getIndex()));360  case MachineOperand::MO_ConstantPoolIndex:361    return LowerSymbolOperand(MO, AsmPrinter.GetCPISymbol(MO.getIndex()));362  case MachineOperand::MO_BlockAddress:363    return LowerSymbolOperand(364        MO, AsmPrinter.GetBlockAddressSymbol(MO.getBlockAddress()));365  case MachineOperand::MO_RegisterMask:366    // Ignore call clobbers.367    return MCOperand();368  }369}370 371// Replace TAILJMP opcodes with their equivalent opcodes that have encoding372// information.373static unsigned convertTailJumpOpcode(unsigned Opcode) {374  switch (Opcode) {375  case X86::TAILJMPr:376    Opcode = X86::JMP32r;377    break;378  case X86::TAILJMPm:379    Opcode = X86::JMP32m;380    break;381  case X86::TAILJMPr64:382    Opcode = X86::JMP64r;383    break;384  case X86::TAILJMPm64:385    Opcode = X86::JMP64m;386    break;387  case X86::TAILJMPr64_REX:388    Opcode = X86::JMP64r_REX;389    break;390  case X86::TAILJMPm64_REX:391    Opcode = X86::JMP64m_REX;392    break;393  case X86::TAILJMPd:394  case X86::TAILJMPd64:395    Opcode = X86::JMP_1;396    break;397  case X86::TAILJMPd_CC:398  case X86::TAILJMPd64_CC:399    Opcode = X86::JCC_1;400    break;401  }402 403  return Opcode;404}405 406void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {407  OutMI.setOpcode(MI->getOpcode());408 409  for (const MachineOperand &MO : MI->operands())410    if (auto Op = LowerMachineOperand(MI, MO); Op.isValid())411      OutMI.addOperand(Op);412 413  bool In64BitMode = AsmPrinter.getSubtarget().is64Bit();414  if (X86::optimizeInstFromVEX3ToVEX2(OutMI, MI->getDesc()) ||415      X86::optimizeShiftRotateWithImmediateOne(OutMI) ||416      X86::optimizeVPCMPWithImmediateOneOrSix(OutMI) ||417      X86::optimizeMOVSX(OutMI) || X86::optimizeINCDEC(OutMI, In64BitMode) ||418      X86::optimizeMOV(OutMI, In64BitMode) ||419      X86::optimizeToFixedRegisterOrShortImmediateForm(OutMI))420    return;421 422  // Handle a few special cases to eliminate operand modifiers.423  switch (OutMI.getOpcode()) {424  case X86::LEA64_32r:425  case X86::LEA64r:426  case X86::LEA16r:427  case X86::LEA32r:428    // LEA should have a segment register, but it must be empty.429    assert(OutMI.getNumOperands() == 1 + X86::AddrNumOperands &&430           "Unexpected # of LEA operands");431    assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&432           "LEA has segment specified!");433    break;434  case X86::MULX32Hrr:435  case X86::MULX32Hrm:436  case X86::MULX64Hrr:437  case X86::MULX64Hrm: {438    // Turn into regular MULX by duplicating the destination.439    unsigned NewOpc;440    switch (OutMI.getOpcode()) {441    default: llvm_unreachable("Invalid opcode");442    case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break;443    case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break;444    case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break;445    case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break;446    }447    OutMI.setOpcode(NewOpc);448    // Duplicate the destination.449    MCRegister DestReg = OutMI.getOperand(0).getReg();450    OutMI.insert(OutMI.begin(), MCOperand::createReg(DestReg));451    break;452  }453  // CALL64r, CALL64pcrel32 - These instructions used to have454  // register inputs modeled as normal uses instead of implicit uses.  As such,455  // they we used to truncate off all but the first operand (the callee). This456  // issue seems to have been fixed at some point. This assert verifies that.457  case X86::CALL64r:458  case X86::CALL64pcrel32:459    assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");460    break;461  case X86::EH_RETURN:462  case X86::EH_RETURN64: {463    OutMI = MCInst();464    OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));465    break;466  }467  case X86::CLEANUPRET: {468    // Replace CLEANUPRET with the appropriate RET.469    OutMI = MCInst();470    OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));471    break;472  }473  case X86::CATCHRET: {474    // Replace CATCHRET with the appropriate RET.475    const X86Subtarget &Subtarget = AsmPrinter.getSubtarget();476    unsigned ReturnReg = In64BitMode ? X86::RAX : X86::EAX;477    OutMI = MCInst();478    OutMI.setOpcode(getRetOpcode(Subtarget));479    OutMI.addOperand(MCOperand::createReg(ReturnReg));480    break;481  }482  // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump483  // instruction.484  case X86::TAILJMPr:485  case X86::TAILJMPr64:486  case X86::TAILJMPr64_REX:487  case X86::TAILJMPd:488  case X86::TAILJMPd64:489    assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");490    OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));491    break;492  case X86::TAILJMPd_CC:493  case X86::TAILJMPd64_CC:494    assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!");495    OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));496    break;497  case X86::TAILJMPm:498  case X86::TAILJMPm64:499  case X86::TAILJMPm64_REX:500    assert(OutMI.getNumOperands() == X86::AddrNumOperands &&501           "Unexpected number of operands!");502    OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));503    break;504  case X86::MASKMOVDQU:505  case X86::VMASKMOVDQU:506    if (In64BitMode)507      OutMI.setFlags(X86::IP_HAS_AD_SIZE);508    break;509  case X86::BSF16rm:510  case X86::BSF16rr:511  case X86::BSF32rm:512  case X86::BSF32rr:513  case X86::BSF64rm:514  case X86::BSF64rr: {515    // Add an REP prefix to BSF instructions so that new processors can516    // recognize as TZCNT, which has better performance than BSF.517    // BSF and TZCNT have different interpretations on ZF bit. So make sure518    // it won't be used later.519    const MachineOperand *FlagDef =520        MI->findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);521    if (!MF.getFunction().hasOptSize() && FlagDef && FlagDef->isDead())522      OutMI.setFlags(X86::IP_HAS_REPEAT);523    break;524  }525  default:526    break;527  }528}529 530void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,531                                 const MachineInstr &MI) {532  NoAutoPaddingScope NoPadScope(*OutStreamer);533  bool Is64Bits = getSubtarget().is64Bit();534  bool Is64BitsLP64 = getSubtarget().isTarget64BitLP64();535  MCContext &Ctx = OutStreamer->getContext();536 537  X86::Specifier Specifier;538  switch (MI.getOpcode()) {539  case X86::TLS_addr32:540  case X86::TLS_addr64:541  case X86::TLS_addrX32:542    Specifier = X86::S_TLSGD;543    break;544  case X86::TLS_base_addr32:545    Specifier = X86::S_TLSLDM;546    break;547  case X86::TLS_base_addr64:548  case X86::TLS_base_addrX32:549    Specifier = X86::S_TLSLD;550    break;551  case X86::TLS_desc32:552  case X86::TLS_desc64:553    Specifier = X86::S_TLSDESC;554    break;555  default:556    llvm_unreachable("unexpected opcode");557  }558 559  const MCSymbolRefExpr *Sym = MCSymbolRefExpr::create(560      MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), Specifier, Ctx);561 562  // Before binutils 2.41, ld has a bogus TLS relaxation error when the GD/LD563  // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is564  // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by565  // only using GOT when GOTPCRELX is enabled.566  // TODO Delete the workaround when rustc no longer relies on the hack567  bool UseGot = MMI->getModule()->getRtLibUseGOT() &&568                Ctx.getTargetOptions()->X86RelaxRelocations;569 570  if (Specifier == X86::S_TLSDESC) {571    const MCSymbolRefExpr *Expr = MCSymbolRefExpr::create(572        MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), X86::S_TLSCALL,573        Ctx);574    EmitAndCountInstruction(575        MCInstBuilder(Is64BitsLP64 ? X86::LEA64r : X86::LEA32r)576            .addReg(Is64BitsLP64 ? X86::RAX : X86::EAX)577            .addReg(Is64Bits ? X86::RIP : X86::EBX)578            .addImm(1)579            .addReg(0)580            .addExpr(Sym)581            .addReg(0));582    EmitAndCountInstruction(583        MCInstBuilder(Is64Bits ? X86::CALL64m : X86::CALL32m)584            .addReg(Is64BitsLP64 ? X86::RAX : X86::EAX)585            .addImm(1)586            .addReg(0)587            .addExpr(Expr)588            .addReg(0));589  } else if (Is64Bits) {590    bool NeedsPadding = Specifier == X86::S_TLSGD;591    if (NeedsPadding && Is64BitsLP64)592      EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));593    EmitAndCountInstruction(MCInstBuilder(X86::LEA64r)594                                .addReg(X86::RDI)595                                .addReg(X86::RIP)596                                .addImm(1)597                                .addReg(0)598                                .addExpr(Sym)599                                .addReg(0));600    const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("__tls_get_addr");601    if (NeedsPadding) {602      if (!UseGot)603        EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));604      EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));605      EmitAndCountInstruction(MCInstBuilder(X86::REX64_PREFIX));606    }607    if (UseGot) {608      const MCExpr *Expr =609          MCSymbolRefExpr::create(TlsGetAddr, X86::S_GOTPCREL, Ctx);610      EmitAndCountInstruction(MCInstBuilder(X86::CALL64m)611                                  .addReg(X86::RIP)612                                  .addImm(1)613                                  .addReg(0)614                                  .addExpr(Expr)615                                  .addReg(0));616    } else {617      EmitAndCountInstruction(618          MCInstBuilder(X86::CALL64pcrel32)619              .addExpr(MCSymbolRefExpr::create(TlsGetAddr, X86::S_PLT, Ctx)));620    }621  } else {622    if (Specifier == X86::S_TLSGD && !UseGot) {623      EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)624                                  .addReg(X86::EAX)625                                  .addReg(0)626                                  .addImm(1)627                                  .addReg(X86::EBX)628                                  .addExpr(Sym)629                                  .addReg(0));630    } else {631      EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)632                                  .addReg(X86::EAX)633                                  .addReg(X86::EBX)634                                  .addImm(1)635                                  .addReg(0)636                                  .addExpr(Sym)637                                  .addReg(0));638    }639 640    const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("___tls_get_addr");641    if (UseGot) {642      const MCExpr *Expr = MCSymbolRefExpr::create(TlsGetAddr, X86::S_GOT, Ctx);643      EmitAndCountInstruction(MCInstBuilder(X86::CALL32m)644                                  .addReg(X86::EBX)645                                  .addImm(1)646                                  .addReg(0)647                                  .addExpr(Expr)648                                  .addReg(0));649    } else {650      EmitAndCountInstruction(651          MCInstBuilder(X86::CALLpcrel32)652              .addExpr(MCSymbolRefExpr::create(TlsGetAddr, X86::S_PLT, Ctx)));653    }654  }655}656 657/// Emit the largest nop instruction smaller than or equal to \p NumBytes658/// bytes.  Return the size of nop emitted.659static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,660                        const X86Subtarget *Subtarget) {661  // Determine the longest nop which can be efficiently decoded for the given662  // target cpu.  15-bytes is the longest single NOP instruction, but some663  // platforms can't decode the longest forms efficiently.664  unsigned MaxNopLength = 1;665  if (Subtarget->is64Bit()) {666    // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the667    // IndexReg/BaseReg below need to be updated.668    if (Subtarget->hasFeature(X86::TuningFast7ByteNOP))669      MaxNopLength = 7;670    else if (Subtarget->hasFeature(X86::TuningFast15ByteNOP))671      MaxNopLength = 15;672    else if (Subtarget->hasFeature(X86::TuningFast11ByteNOP))673      MaxNopLength = 11;674    else675      MaxNopLength = 10;676  } if (Subtarget->is32Bit())677    MaxNopLength = 2;678 679  // Cap a single nop emission at the profitable value for the target680  NumBytes = std::min(NumBytes, MaxNopLength);681 682  unsigned NopSize;683  unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg;684  IndexReg = Displacement = SegmentReg = 0;685  BaseReg = X86::RAX;686  ScaleVal = 1;687  switch (NumBytes) {688  case 0:689    llvm_unreachable("Zero nops?");690    break;691  case 1:692    NopSize = 1;693    Opc = X86::NOOP;694    break;695  case 2:696    NopSize = 2;697    Opc = X86::XCHG16ar;698    break;699  case 3:700    NopSize = 3;701    Opc = X86::NOOPL;702    break;703  case 4:704    NopSize = 4;705    Opc = X86::NOOPL;706    Displacement = 8;707    break;708  case 5:709    NopSize = 5;710    Opc = X86::NOOPL;711    Displacement = 8;712    IndexReg = X86::RAX;713    break;714  case 6:715    NopSize = 6;716    Opc = X86::NOOPW;717    Displacement = 8;718    IndexReg = X86::RAX;719    break;720  case 7:721    NopSize = 7;722    Opc = X86::NOOPL;723    Displacement = 512;724    break;725  case 8:726    NopSize = 8;727    Opc = X86::NOOPL;728    Displacement = 512;729    IndexReg = X86::RAX;730    break;731  case 9:732    NopSize = 9;733    Opc = X86::NOOPW;734    Displacement = 512;735    IndexReg = X86::RAX;736    break;737  default:738    NopSize = 10;739    Opc = X86::NOOPW;740    Displacement = 512;741    IndexReg = X86::RAX;742    SegmentReg = X86::CS;743    break;744  }745 746  unsigned NumPrefixes = std::min(NumBytes - NopSize, 5U);747  NopSize += NumPrefixes;748  for (unsigned i = 0; i != NumPrefixes; ++i)749    OS.emitBytes("\x66");750 751  switch (Opc) {752  default: llvm_unreachable("Unexpected opcode");753  case X86::NOOP:754    OS.emitInstruction(MCInstBuilder(Opc), *Subtarget);755    break;756  case X86::XCHG16ar:757    OS.emitInstruction(MCInstBuilder(Opc).addReg(X86::AX).addReg(X86::AX),758                       *Subtarget);759    break;760  case X86::NOOPL:761  case X86::NOOPW:762    OS.emitInstruction(MCInstBuilder(Opc)763                           .addReg(BaseReg)764                           .addImm(ScaleVal)765                           .addReg(IndexReg)766                           .addImm(Displacement)767                           .addReg(SegmentReg),768                       *Subtarget);769    break;770  }771  assert(NopSize <= NumBytes && "We overemitted?");772  return NopSize;773}774 775/// Emit the optimal amount of multi-byte nops on X86.776static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,777                        const X86Subtarget *Subtarget) {778  unsigned NopsToEmit = NumBytes;779  (void)NopsToEmit;780  while (NumBytes) {781    NumBytes -= emitNop(OS, NumBytes, Subtarget);782    assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!");783  }784}785 786void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,787                                    X86MCInstLower &MCIL) {788  assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");789 790  NoAutoPaddingScope NoPadScope(*OutStreamer);791 792  StatepointOpers SOpers(&MI);793  if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {794    emitX86Nops(*OutStreamer, PatchBytes, Subtarget);795  } else {796    // Lower call target and choose correct opcode797    const MachineOperand &CallTarget = SOpers.getCallTarget();798    MCOperand CallTargetMCOp;799    unsigned CallOpcode;800    switch (CallTarget.getType()) {801    case MachineOperand::MO_GlobalAddress:802    case MachineOperand::MO_ExternalSymbol:803      CallTargetMCOp = MCIL.LowerSymbolOperand(804          CallTarget, MCIL.GetSymbolFromOperand(CallTarget));805      CallOpcode = X86::CALL64pcrel32;806      // Currently, we only support relative addressing with statepoints.807      // Otherwise, we'll need a scratch register to hold the target808      // address.  You'll fail asserts during load & relocation if this809      // symbol is to far away. (TODO: support non-relative addressing)810      break;811    case MachineOperand::MO_Immediate:812      CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());813      CallOpcode = X86::CALL64pcrel32;814      // Currently, we only support relative addressing with statepoints.815      // Otherwise, we'll need a scratch register to hold the target816      // immediate.  You'll fail asserts during load & relocation if this817      // address is to far away. (TODO: support non-relative addressing)818      break;819    case MachineOperand::MO_Register:820      // FIXME: Add retpoline support and remove this.821      if (Subtarget->useIndirectThunkCalls())822        report_fatal_error("Lowering register statepoints with thunks not "823                           "yet implemented.");824      CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());825      CallOpcode = X86::CALL64r;826      break;827    default:828      llvm_unreachable("Unsupported operand type in statepoint call target");829      break;830    }831 832    // Emit call833    MCInst CallInst;834    CallInst.setOpcode(CallOpcode);835    CallInst.addOperand(CallTargetMCOp);836    OutStreamer->emitInstruction(CallInst, getSubtargetInfo());837    maybeEmitNopAfterCallForWindowsEH(&MI);838  }839 840  // Record our statepoint node in the same section used by STACKMAP841  // and PATCHPOINT842  auto &Ctx = OutStreamer->getContext();843  MCSymbol *MILabel = Ctx.createTempSymbol();844  OutStreamer->emitLabel(MILabel);845  SM.recordStatepoint(*MILabel, MI);846}847 848void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI,849                                     X86MCInstLower &MCIL) {850  // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,851  //                  <opcode>, <operands>852 853  NoAutoPaddingScope NoPadScope(*OutStreamer);854 855  Register DefRegister = FaultingMI.getOperand(0).getReg();856  FaultMaps::FaultKind FK =857      static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());858  MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();859  unsigned Opcode = FaultingMI.getOperand(3).getImm();860  unsigned OperandsBeginIdx = 4;861 862  auto &Ctx = OutStreamer->getContext();863  MCSymbol *FaultingLabel = Ctx.createTempSymbol();864  OutStreamer->emitLabel(FaultingLabel);865 866  assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");867  FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);868 869  MCInst MI;870  MI.setOpcode(Opcode);871 872  if (DefRegister != X86::NoRegister)873    MI.addOperand(MCOperand::createReg(DefRegister));874 875  for (const MachineOperand &MO :876       llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx))877    if (auto Op = MCIL.LowerMachineOperand(&FaultingMI, MO); Op.isValid())878      MI.addOperand(Op);879 880  OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());881  OutStreamer->emitInstruction(MI, getSubtargetInfo());882}883 884void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,885                                     X86MCInstLower &MCIL) {886  bool Is64Bits = Subtarget->is64Bit();887  MCContext &Ctx = OutStreamer->getContext();888  MCSymbol *fentry = Ctx.getOrCreateSymbol("__fentry__");889  const MCSymbolRefExpr *Op = MCSymbolRefExpr::create(fentry, Ctx);890 891  EmitAndCountInstruction(892      MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32)893          .addExpr(Op));894}895 896void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {897  assert(std::next(MI.getIterator())->isCall() &&898         "KCFI_CHECK not followed by a call instruction");899 900  // Adjust the offset for patchable-function-prefix. X86InstrInfo::getNop()901  // returns a 1-byte X86::NOOP, which means the offset is the same in902  // bytes.  This assumes that patchable-function-prefix is the same for all903  // functions.904  const MachineFunction &MF = *MI.getMF();905  int64_t PrefixNops = 0;906  (void)MF.getFunction()907      .getFnAttribute("patchable-function-prefix")908      .getValueAsString()909      .getAsInteger(10, PrefixNops);910 911  // KCFI allows indirect calls to any location that's preceded by a valid912  // type identifier. To avoid encoding the full constant into an instruction,913  // and thus emitting potential call target gadgets at each indirect call914  // site, load a negated constant to a register and compare that to the915  // expected value at the call target.916  const Register AddrReg = MI.getOperand(0).getReg();917  const uint32_t Type = MI.getOperand(1).getImm();918  // The check is immediately before the call. If the call target is in R10,919  // we can clobber R11 for the check instead.920  unsigned TempReg = AddrReg == X86::R10 ? X86::R11D : X86::R10D;921  EmitAndCountInstruction(922      MCInstBuilder(X86::MOV32ri).addReg(TempReg).addImm(-MaskKCFIType(Type)));923  EmitAndCountInstruction(MCInstBuilder(X86::ADD32rm)924                              .addReg(X86::NoRegister)925                              .addReg(TempReg)926                              .addReg(AddrReg)927                              .addImm(1)928                              .addReg(X86::NoRegister)929                              .addImm(-(PrefixNops + 4))930                              .addReg(X86::NoRegister));931 932  MCSymbol *Pass = OutContext.createTempSymbol();933  EmitAndCountInstruction(934      MCInstBuilder(X86::JCC_1)935          .addExpr(MCSymbolRefExpr::create(Pass, OutContext))936          .addImm(X86::COND_E));937 938  MCSymbol *Trap = OutContext.createTempSymbol();939  OutStreamer->emitLabel(Trap);940  EmitAndCountInstruction(MCInstBuilder(X86::TRAP));941  emitKCFITrapEntry(MF, Trap);942  OutStreamer->emitLabel(Pass);943}944 945void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {946  // FIXME: Make this work on non-ELF.947  if (!TM.getTargetTriple().isOSBinFormatELF()) {948    report_fatal_error("llvm.asan.check.memaccess only supported on ELF");949    return;950  }951 952  const auto &Reg = MI.getOperand(0).getReg();953  ASanAccessInfo AccessInfo(MI.getOperand(1).getImm());954 955  uint64_t ShadowBase;956  int MappingScale;957  bool OrShadowOffset;958  getAddressSanitizerParams(TM.getTargetTriple(), 64, AccessInfo.CompileKernel,959                            &ShadowBase, &MappingScale, &OrShadowOffset);960 961  StringRef Name = AccessInfo.IsWrite ? "store" : "load";962  StringRef Op = OrShadowOffset ? "or" : "add";963  std::string SymName = ("__asan_check_" + Name + "_" + Op + "_" +964                         Twine(1ULL << AccessInfo.AccessSizeIndex) + "_" +965                         TM.getMCRegisterInfo()->getName(Reg.asMCReg()))966                            .str();967  if (OrShadowOffset)968    report_fatal_error(969        "OrShadowOffset is not supported with optimized callbacks");970 971  EmitAndCountInstruction(972      MCInstBuilder(X86::CALL64pcrel32)973          .addExpr(MCSymbolRefExpr::create(974              OutContext.getOrCreateSymbol(SymName), OutContext)));975}976 977void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,978                                      X86MCInstLower &MCIL) {979  // PATCHABLE_OP minsize980 981  NoAutoPaddingScope NoPadScope(*OutStreamer);982 983  auto NextMI = std::find_if(std::next(MI.getIterator()),984                             MI.getParent()->end().getInstrIterator(),985                             [](auto &II) { return !II.isMetaInstruction(); });986 987  SmallString<256> Code;988  unsigned MinSize = MI.getOperand(0).getImm();989 990  if (NextMI != MI.getParent()->end() && !NextMI->isInlineAsm()) {991    // Lower the next MachineInstr to find its byte size.992    // If the next instruction is inline assembly, we skip lowering it for now,993    // and assume we should always generate NOPs.994    MCInst MCI;995    MCIL.Lower(&*NextMI, MCI);996 997    SmallVector<MCFixup, 4> Fixups;998    CodeEmitter->encodeInstruction(MCI, Code, Fixups, getSubtargetInfo());999  }1000 1001  if (Code.size() < MinSize) {1002    if (MinSize == 2 && Subtarget->is32Bit() &&1003        Subtarget->isTargetWindowsMSVC() &&1004        (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) {1005      // For compatibility reasons, when targetting MSVC, it is important to1006      // generate a 'legacy' NOP in the form of a 8B FF MOV EDI, EDI. Some tools1007      // rely specifically on this pattern to be able to patch a function.1008      // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE.1009      OutStreamer->emitInstruction(1010          MCInstBuilder(X86::MOV32rr_REV).addReg(X86::EDI).addReg(X86::EDI),1011          *Subtarget);1012    } else {1013      unsigned NopSize = emitNop(*OutStreamer, MinSize, Subtarget);1014      assert(NopSize == MinSize && "Could not implement MinSize!");1015      (void)NopSize;1016    }1017  }1018}1019 1020// Lower a stackmap of the form:1021// <id>, <shadowBytes>, ...1022void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {1023  SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());1024 1025  auto &Ctx = OutStreamer->getContext();1026  MCSymbol *MILabel = Ctx.createTempSymbol();1027  OutStreamer->emitLabel(MILabel);1028 1029  SM.recordStackMap(*MILabel, MI);1030  unsigned NumShadowBytes = MI.getOperand(1).getImm();1031  SMShadowTracker.reset(NumShadowBytes);1032}1033 1034// Lower a patchpoint of the form:1035// [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...1036void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,1037                                    X86MCInstLower &MCIL) {1038  assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");1039 1040  SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());1041 1042  NoAutoPaddingScope NoPadScope(*OutStreamer);1043 1044  auto &Ctx = OutStreamer->getContext();1045  MCSymbol *MILabel = Ctx.createTempSymbol();1046  OutStreamer->emitLabel(MILabel);1047  SM.recordPatchPoint(*MILabel, MI);1048 1049  PatchPointOpers opers(&MI);1050  unsigned ScratchIdx = opers.getNextScratchIdx();1051  unsigned EncodedBytes = 0;1052  const MachineOperand &CalleeMO = opers.getCallTarget();1053 1054  // Check for null target. If target is non-null (i.e. is non-zero or is1055  // symbolic) then emit a call.1056  if (!(CalleeMO.isImm() && !CalleeMO.getImm())) {1057    MCOperand CalleeMCOp;1058    switch (CalleeMO.getType()) {1059    default:1060      /// FIXME: Add a verifier check for bad callee types.1061      llvm_unreachable("Unrecognized callee operand type.");1062    case MachineOperand::MO_Immediate:1063      if (CalleeMO.getImm())1064        CalleeMCOp = MCOperand::createImm(CalleeMO.getImm());1065      break;1066    case MachineOperand::MO_ExternalSymbol:1067    case MachineOperand::MO_GlobalAddress:1068      CalleeMCOp = MCIL.LowerSymbolOperand(CalleeMO,1069                                           MCIL.GetSymbolFromOperand(CalleeMO));1070      break;1071    }1072 1073    // Emit MOV to materialize the target address and the CALL to target.1074    // This is encoded with 12-13 bytes, depending on which register is used.1075    Register ScratchReg = MI.getOperand(ScratchIdx).getReg();1076    if (X86II::isX86_64ExtendedReg(ScratchReg))1077      EncodedBytes = 13;1078    else1079      EncodedBytes = 12;1080 1081    EmitAndCountInstruction(1082        MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp));1083    // FIXME: Add retpoline support and remove this.1084    if (Subtarget->useIndirectThunkCalls())1085      report_fatal_error(1086          "Lowering patchpoint with thunks not yet implemented.");1087    EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg));1088  }1089 1090  // Emit padding.1091  unsigned NumBytes = opers.getNumPatchBytes();1092  assert(NumBytes >= EncodedBytes &&1093         "Patchpoint can't request size less than the length of a call.");1094 1095  emitX86Nops(*OutStreamer, NumBytes - EncodedBytes, Subtarget);1096}1097 1098void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,1099                                              X86MCInstLower &MCIL) {1100  assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64");1101 1102  NoAutoPaddingScope NoPadScope(*OutStreamer);1103 1104  // We want to emit the following pattern, which follows the x86 calling1105  // convention to prepare for the trampoline call to be patched in.1106  //1107  //   .p2align 1, ...1108  // .Lxray_event_sled_N:1109  //   jmp +N                        // jump across the instrumentation sled1110  //   ...                           // set up arguments in register1111  //   callq __xray_CustomEvent@plt  // force dependency to symbol1112  //   ...1113  //   <jump here>1114  //1115  // After patching, it would look something like:1116  //1117  //   nopw (2-byte nop)1118  //   ...1119  //   callq __xrayCustomEvent  // already lowered1120  //   ...1121  //1122  // ---1123  // First we emit the label and the jump.1124  auto CurSled = OutContext.createTempSymbol("xray_event_sled_", true);1125  OutStreamer->AddComment("# XRay Custom Event Log");1126  OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());1127  OutStreamer->emitLabel(CurSled);1128 1129  // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as1130  // an operand (computed as an offset from the jmp instruction).1131  // FIXME: Find another less hacky way do force the relative jump.1132  OutStreamer->emitBinaryData("\xeb\x0f");1133 1134  // The default C calling convention will place two arguments into %rcx and1135  // %rdx -- so we only work with those.1136  const Register DestRegs[] = {X86::RDI, X86::RSI};1137  bool UsedMask[] = {false, false};1138  // Filled out in loop.1139  Register SrcRegs[] = {0, 0};1140 1141  // Then we put the operands in the %rdi and %rsi registers. We spill the1142  // values in the register before we clobber them, and mark them as used in1143  // UsedMask. In case the arguments are already in the correct register, we use1144  // emit nops appropriately sized to keep the sled the same size in every1145  // situation.1146  for (unsigned I = 0; I < MI.getNumOperands(); ++I)1147    if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I));1148        Op.isValid()) {1149      assert(Op.isReg() && "Only support arguments in registers");1150      SrcRegs[I] = getX86SubSuperRegister(Op.getReg(), 64);1151      assert(SrcRegs[I].isValid() && "Invalid operand");1152      if (SrcRegs[I] != DestRegs[I]) {1153        UsedMask[I] = true;1154        EmitAndCountInstruction(1155            MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));1156      } else {1157        emitX86Nops(*OutStreamer, 4, Subtarget);1158      }1159    }1160 1161  // Now that the register values are stashed, mov arguments into place.1162  // FIXME: This doesn't work if one of the later SrcRegs is equal to an1163  // earlier DestReg. We will have already overwritten over the register before1164  // we can copy from it.1165  for (unsigned I = 0; I < MI.getNumOperands(); ++I)1166    if (SrcRegs[I] != DestRegs[I])1167      EmitAndCountInstruction(1168          MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));1169 1170  // We emit a hard dependency on the __xray_CustomEvent symbol, which is the1171  // name of the trampoline to be implemented by the XRay runtime.1172  auto TSym = OutContext.getOrCreateSymbol("__xray_CustomEvent");1173  MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym);1174  if (isPositionIndependent())1175    TOp.setTargetFlags(X86II::MO_PLT);1176 1177  // Emit the call instruction.1178  EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)1179                              .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));1180 1181  // Restore caller-saved and used registers.1182  for (unsigned I = sizeof UsedMask; I-- > 0;)1183    if (UsedMask[I])1184      EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));1185    else1186      emitX86Nops(*OutStreamer, 1, Subtarget);1187 1188  OutStreamer->AddComment("xray custom event end.");1189 1190  // Record the sled version. Version 0 of this sled was spelled differently, so1191  // we let the runtime handle the different offsets we're using. Version 21192  // changed the absolute address to a PC-relative address.1193  recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);1194}1195 1196void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI,1197                                                    X86MCInstLower &MCIL) {1198  assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64");1199 1200  NoAutoPaddingScope NoPadScope(*OutStreamer);1201 1202  // We want to emit the following pattern, which follows the x86 calling1203  // convention to prepare for the trampoline call to be patched in.1204  //1205  //   .p2align 1, ...1206  // .Lxray_event_sled_N:1207  //   jmp +N                        // jump across the instrumentation sled1208  //   ...                           // set up arguments in register1209  //   callq __xray_TypedEvent@plt  // force dependency to symbol1210  //   ...1211  //   <jump here>1212  //1213  // After patching, it would look something like:1214  //1215  //   nopw (2-byte nop)1216  //   ...1217  //   callq __xrayTypedEvent  // already lowered1218  //   ...1219  //1220  // ---1221  // First we emit the label and the jump.1222  auto CurSled = OutContext.createTempSymbol("xray_typed_event_sled_", true);1223  OutStreamer->AddComment("# XRay Typed Event Log");1224  OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());1225  OutStreamer->emitLabel(CurSled);1226 1227  // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as1228  // an operand (computed as an offset from the jmp instruction).1229  // FIXME: Find another less hacky way do force the relative jump.1230  OutStreamer->emitBinaryData("\xeb\x14");1231 1232  // An x86-64 convention may place three arguments into %rcx, %rdx, and R8,1233  // so we'll work with those. Or we may be called via SystemV, in which case1234  // we don't have to do any translation.1235  const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX};1236  bool UsedMask[] = {false, false, false};1237 1238  // Will fill out src regs in the loop.1239  Register SrcRegs[] = {0, 0, 0};1240 1241  // Then we put the operands in the SystemV registers. We spill the values in1242  // the registers before we clobber them, and mark them as used in UsedMask.1243  // In case the arguments are already in the correct register, we emit nops1244  // appropriately sized to keep the sled the same size in every situation.1245  for (unsigned I = 0; I < MI.getNumOperands(); ++I)1246    if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I));1247        Op.isValid()) {1248      // TODO: Is register only support adequate?1249      assert(Op.isReg() && "Only supports arguments in registers");1250      SrcRegs[I] = getX86SubSuperRegister(Op.getReg(), 64);1251      assert(SrcRegs[I].isValid() && "Invalid operand");1252      if (SrcRegs[I] != DestRegs[I]) {1253        UsedMask[I] = true;1254        EmitAndCountInstruction(1255            MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));1256      } else {1257        emitX86Nops(*OutStreamer, 4, Subtarget);1258      }1259    }1260 1261  // In the above loop we only stash all of the destination registers or emit1262  // nops if the arguments are already in the right place. Doing the actually1263  // moving is postponed until after all the registers are stashed so nothing1264  // is clobbers. We've already added nops to account for the size of mov and1265  // push if the register is in the right place, so we only have to worry about1266  // emitting movs.1267  // FIXME: This doesn't work if one of the later SrcRegs is equal to an1268  // earlier DestReg. We will have already overwritten over the register before1269  // we can copy from it.1270  for (unsigned I = 0; I < MI.getNumOperands(); ++I)1271    if (UsedMask[I])1272      EmitAndCountInstruction(1273          MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));1274 1275  // We emit a hard dependency on the __xray_TypedEvent symbol, which is the1276  // name of the trampoline to be implemented by the XRay runtime.1277  auto TSym = OutContext.getOrCreateSymbol("__xray_TypedEvent");1278  MachineOperand TOp = MachineOperand::CreateMCSymbol(TSym);1279  if (isPositionIndependent())1280    TOp.setTargetFlags(X86II::MO_PLT);1281 1282  // Emit the call instruction.1283  EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)1284                              .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));1285 1286  // Restore caller-saved and used registers.1287  for (unsigned I = sizeof UsedMask; I-- > 0;)1288    if (UsedMask[I])1289      EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));1290    else1291      emitX86Nops(*OutStreamer, 1, Subtarget);1292 1293  OutStreamer->AddComment("xray typed event end.");1294 1295  // Record the sled version.1296  recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);1297}1298 1299void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,1300                                                  X86MCInstLower &MCIL) {1301 1302  NoAutoPaddingScope NoPadScope(*OutStreamer);1303 1304  const Function &F = MF->getFunction();1305  if (F.hasFnAttribute("patchable-function-entry")) {1306    unsigned Num;1307    if (F.getFnAttribute("patchable-function-entry")1308            .getValueAsString()1309            .getAsInteger(10, Num))1310      return;1311    emitX86Nops(*OutStreamer, Num, Subtarget);1312    return;1313  }1314  // We want to emit the following pattern:1315  //1316  //   .p2align 1, ...1317  // .Lxray_sled_N:1318  //   jmp .tmpN1319  //   # 9 bytes worth of noops1320  //1321  // We need the 9 bytes because at runtime, we'd be patching over the full 111322  // bytes with the following pattern:1323  //1324  //   mov %r10, <function id, 32-bit>   // 6 bytes1325  //   call <relative offset, 32-bits>   // 5 bytes1326  //1327  auto CurSled = OutContext.createTempSymbol("xray_sled_", true);1328  OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());1329  OutStreamer->emitLabel(CurSled);1330 1331  // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as1332  // an operand (computed as an offset from the jmp instruction).1333  // FIXME: Find another less hacky way do force the relative jump.1334  OutStreamer->emitBytes("\xeb\x09");1335  emitX86Nops(*OutStreamer, 9, Subtarget);1336  recordSled(CurSled, MI, SledKind::FUNCTION_ENTER, 2);1337}1338 1339void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,1340                                       X86MCInstLower &MCIL) {1341  NoAutoPaddingScope NoPadScope(*OutStreamer);1342 1343  // Since PATCHABLE_RET takes the opcode of the return statement as an1344  // argument, we use that to emit the correct form of the RET that we want.1345  // i.e. when we see this:1346  //1347  //   PATCHABLE_RET X86::RET ...1348  //1349  // We should emit the RET followed by sleds.1350  //1351  //   .p2align 1, ...1352  // .Lxray_sled_N:1353  //   ret  # or equivalent instruction1354  //   # 10 bytes worth of noops1355  //1356  // This just makes sure that the alignment for the next instruction is 2.1357  auto CurSled = OutContext.createTempSymbol("xray_sled_", true);1358  OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());1359  OutStreamer->emitLabel(CurSled);1360  unsigned OpCode = MI.getOperand(0).getImm();1361  MCInst Ret;1362  Ret.setOpcode(OpCode);1363  for (auto &MO : drop_begin(MI.operands()))1364    if (auto Op = MCIL.LowerMachineOperand(&MI, MO); Op.isValid())1365      Ret.addOperand(Op);1366  OutStreamer->emitInstruction(Ret, getSubtargetInfo());1367  emitX86Nops(*OutStreamer, 10, Subtarget);1368  recordSled(CurSled, MI, SledKind::FUNCTION_EXIT, 2);1369}1370 1371void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI,1372                                             X86MCInstLower &MCIL) {1373  MCInst TC;1374  TC.setOpcode(convertTailJumpOpcode(MI.getOperand(0).getImm()));1375  // Drop the tail jump opcode.1376  auto TCOperands = drop_begin(MI.operands());1377  bool IsConditional = TC.getOpcode() == X86::JCC_1;1378  MCSymbol *FallthroughLabel;1379  if (IsConditional) {1380    // Rewrite:1381    //   je target1382    //1383    // To:1384    //   jne .fallthrough1385    //   .p2align 1, ...1386    // .Lxray_sled_N:1387    //   SLED_CODE1388    //   jmp target1389    // .fallthrough:1390    FallthroughLabel = OutContext.createTempSymbol();1391    EmitToStreamer(1392        *OutStreamer,1393        MCInstBuilder(X86::JCC_1)1394            .addExpr(MCSymbolRefExpr::create(FallthroughLabel, OutContext))1395            .addImm(X86::GetOppositeBranchCondition(1396                static_cast<X86::CondCode>(MI.getOperand(2).getImm()))));1397    TC.setOpcode(X86::JMP_1);1398    // Drop the condition code.1399    TCOperands = drop_end(TCOperands);1400  }1401 1402  NoAutoPaddingScope NoPadScope(*OutStreamer);1403 1404  // Like PATCHABLE_RET, we have the actual instruction in the operands to this1405  // instruction so we lower that particular instruction and its operands.1406  // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how1407  // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to1408  // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual1409  // tail call much like how we have it in PATCHABLE_RET.1410  auto CurSled = OutContext.createTempSymbol("xray_sled_", true);1411  OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());1412  OutStreamer->emitLabel(CurSled);1413  auto Target = OutContext.createTempSymbol();1414 1415  // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as1416  // an operand (computed as an offset from the jmp instruction).1417  // FIXME: Find another less hacky way do force the relative jump.1418  OutStreamer->emitBytes("\xeb\x09");1419  emitX86Nops(*OutStreamer, 9, Subtarget);1420  OutStreamer->emitLabel(Target);1421  recordSled(CurSled, MI, SledKind::TAIL_CALL, 2);1422 1423  // Before emitting the instruction, add a comment to indicate that this is1424  // indeed a tail call.1425  OutStreamer->AddComment("TAILCALL");1426  for (auto &MO : TCOperands)1427    if (auto Op = MCIL.LowerMachineOperand(&MI, MO); Op.isValid())1428      TC.addOperand(Op);1429  OutStreamer->emitInstruction(TC, getSubtargetInfo());1430 1431  if (IsConditional)1432    OutStreamer->emitLabel(FallthroughLabel);1433}1434 1435static unsigned getSrcIdx(const MachineInstr* MI, unsigned SrcIdx) {1436  if (X86II::isKMasked(MI->getDesc().TSFlags)) {1437    // Skip mask operand.1438    ++SrcIdx;1439    if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {1440      // Skip passthru operand.1441      ++SrcIdx;1442    }1443  }1444  return SrcIdx;1445}1446 1447static void printDstRegisterName(raw_ostream &CS, const MachineInstr *MI,1448                                 unsigned SrcOpIdx) {1449  const MachineOperand &DstOp = MI->getOperand(0);1450  CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg());1451 1452  // Handle AVX512 MASK/MASXZ write mask comments.1453  // MASK: zmmX {%kY}1454  // MASKZ: zmmX {%kY} {z}1455  if (X86II::isKMasked(MI->getDesc().TSFlags)) {1456    const MachineOperand &WriteMaskOp = MI->getOperand(SrcOpIdx - 1);1457    StringRef Mask = X86ATTInstPrinter::getRegisterName(WriteMaskOp.getReg());1458    CS << " {%" << Mask << "}";1459    if (!X86II::isKMergeMasked(MI->getDesc().TSFlags)) {1460      CS << " {z}";1461    }1462  }1463}1464 1465static void printShuffleMask(raw_ostream &CS, StringRef Src1Name,1466                             StringRef Src2Name, ArrayRef<int> Mask) {1467  // One source operand, fix the mask to print all elements in one span.1468  SmallVector<int, 8> ShuffleMask(Mask);1469  if (Src1Name == Src2Name)1470    for (int i = 0, e = ShuffleMask.size(); i != e; ++i)1471      if (ShuffleMask[i] >= e)1472        ShuffleMask[i] -= e;1473 1474  for (int i = 0, e = ShuffleMask.size(); i != e; ++i) {1475    if (i != 0)1476      CS << ",";1477    if (ShuffleMask[i] == SM_SentinelZero) {1478      CS << "zero";1479      continue;1480    }1481 1482    // Otherwise, it must come from src1 or src2.  Print the span of elements1483    // that comes from this src.1484    bool isSrc1 = ShuffleMask[i] < (int)e;1485    CS << (isSrc1 ? Src1Name : Src2Name) << '[';1486 1487    bool IsFirst = true;1488    while (i != e && ShuffleMask[i] != SM_SentinelZero &&1489           (ShuffleMask[i] < (int)e) == isSrc1) {1490      if (!IsFirst)1491        CS << ',';1492      else1493        IsFirst = false;1494      if (ShuffleMask[i] == SM_SentinelUndef)1495        CS << "u";1496      else1497        CS << ShuffleMask[i] % (int)e;1498      ++i;1499    }1500    CS << ']';1501    --i; // For loop increments element #.1502  }1503}1504 1505static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx,1506                                     unsigned SrcOp2Idx, ArrayRef<int> Mask) {1507  std::string Comment;1508 1509  const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx);1510  const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx);1511  StringRef Src1Name = SrcOp1.isReg()1512                           ? X86ATTInstPrinter::getRegisterName(SrcOp1.getReg())1513                           : "mem";1514  StringRef Src2Name = SrcOp2.isReg()1515                           ? X86ATTInstPrinter::getRegisterName(SrcOp2.getReg())1516                           : "mem";1517 1518  raw_string_ostream CS(Comment);1519  printDstRegisterName(CS, MI, SrcOp1Idx);1520  CS << " = ";1521  printShuffleMask(CS, Src1Name, Src2Name, Mask);1522 1523  return Comment;1524}1525 1526static void printConstant(const APInt &Val, raw_ostream &CS,1527                          bool PrintZero = false) {1528  if (Val.getBitWidth() <= 64) {1529    CS << (PrintZero ? 0ULL : Val.getZExtValue());1530  } else {1531    // print multi-word constant as (w0,w1)1532    CS << "(";1533    for (int i = 0, N = Val.getNumWords(); i < N; ++i) {1534      if (i > 0)1535        CS << ",";1536      CS << (PrintZero ? 0ULL : Val.getRawData()[i]);1537    }1538    CS << ")";1539  }1540}1541 1542static void printConstant(const APFloat &Flt, raw_ostream &CS,1543                          bool PrintZero = false) {1544  SmallString<32> Str;1545  // Force scientific notation to distinguish from integers.1546  if (PrintZero)1547    APFloat::getZero(Flt.getSemantics()).toString(Str, 0, 0);1548  else1549    Flt.toString(Str, 0, 0);1550  CS << Str;1551}1552 1553static void printConstant(const Constant *COp, unsigned BitWidth,1554                          raw_ostream &CS, bool PrintZero = false) {1555  if (isa<UndefValue>(COp)) {1556    CS << "u";1557  } else if (auto *CI = dyn_cast<ConstantInt>(COp)) {1558    if (auto VTy = dyn_cast<FixedVectorType>(CI->getType())) {1559      for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I) {1560        if (I != 0)1561          CS << ',';1562        printConstant(CI->getValue(), CS, PrintZero);1563      }1564    } else1565      printConstant(CI->getValue(), CS, PrintZero);1566  } else if (auto *CF = dyn_cast<ConstantFP>(COp)) {1567    if (auto VTy = dyn_cast<FixedVectorType>(CF->getType())) {1568      for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I) {1569        if (I != 0)1570          CS << ',';1571        printConstant(CF->getValueAPF(), CS, PrintZero);1572      }1573    } else1574      printConstant(CF->getValueAPF(), CS, PrintZero);1575  } else if (auto *CDS = dyn_cast<ConstantDataSequential>(COp)) {1576    Type *EltTy = CDS->getElementType();1577    bool IsInteger = EltTy->isIntegerTy();1578    bool IsFP = EltTy->isHalfTy() || EltTy->isFloatTy() || EltTy->isDoubleTy();1579    unsigned EltBits = EltTy->getPrimitiveSizeInBits();1580    unsigned E = std::min(BitWidth / EltBits, (unsigned)CDS->getNumElements());1581    if ((BitWidth % EltBits) == 0) {1582      for (unsigned I = 0; I != E; ++I) {1583        if (I != 0)1584          CS << ",";1585        if (IsInteger)1586          printConstant(CDS->getElementAsAPInt(I), CS, PrintZero);1587        else if (IsFP)1588          printConstant(CDS->getElementAsAPFloat(I), CS, PrintZero);1589        else1590          CS << "?";1591      }1592    } else {1593      CS << "?";1594    }1595  } else if (auto *CV = dyn_cast<ConstantVector>(COp)) {1596    unsigned EltBits = CV->getType()->getScalarSizeInBits();1597    unsigned E = std::min(BitWidth / EltBits, CV->getNumOperands());1598    if ((BitWidth % EltBits) == 0) {1599      for (unsigned I = 0; I != E; ++I) {1600        if (I != 0)1601          CS << ",";1602        printConstant(CV->getOperand(I), EltBits, CS, PrintZero);1603      }1604    } else {1605      CS << "?";1606    }1607  } else {1608    CS << "?";1609  }1610}1611 1612static void printZeroUpperMove(const MachineInstr *MI, MCStreamer &OutStreamer,1613                               int SclWidth, int VecWidth,1614                               const char *ShuffleComment) {1615  unsigned SrcIdx = getSrcIdx(MI, 1);1616 1617  std::string Comment;1618  raw_string_ostream CS(Comment);1619  printDstRegisterName(CS, MI, SrcIdx);1620  CS << " = ";1621 1622  if (auto *C = X86::getConstantFromPool(*MI, SrcIdx)) {1623    CS << "[";1624    printConstant(C, SclWidth, CS);1625    for (int I = 1, E = VecWidth / SclWidth; I < E; ++I) {1626      CS << ",";1627      printConstant(C, SclWidth, CS, true);1628    }1629    CS << "]";1630    OutStreamer.AddComment(CS.str());1631    return; // early-out1632  }1633 1634  // We didn't find a constant load, fallback to a shuffle mask decode.1635  CS << ShuffleComment;1636  OutStreamer.AddComment(CS.str());1637}1638 1639static void printBroadcast(const MachineInstr *MI, MCStreamer &OutStreamer,1640                           int Repeats, int BitWidth) {1641  unsigned SrcIdx = getSrcIdx(MI, 1);1642  if (auto *C = X86::getConstantFromPool(*MI, SrcIdx)) {1643    std::string Comment;1644    raw_string_ostream CS(Comment);1645    printDstRegisterName(CS, MI, SrcIdx);1646    CS << " = [";1647    for (int l = 0; l != Repeats; ++l) {1648      if (l != 0)1649        CS << ",";1650      printConstant(C, BitWidth, CS);1651    }1652    CS << "]";1653    OutStreamer.AddComment(CS.str());1654  }1655}1656 1657static bool printExtend(const MachineInstr *MI, MCStreamer &OutStreamer,1658                        int SrcEltBits, int DstEltBits, bool IsSext) {1659  unsigned SrcIdx = getSrcIdx(MI, 1);1660  auto *C = X86::getConstantFromPool(*MI, SrcIdx);1661  if (C && C->getType()->getScalarSizeInBits() == unsigned(SrcEltBits)) {1662    if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {1663      int NumElts = CDS->getNumElements();1664      std::string Comment;1665      raw_string_ostream CS(Comment);1666      printDstRegisterName(CS, MI, SrcIdx);1667      CS << " = [";1668      for (int i = 0; i != NumElts; ++i) {1669        if (i != 0)1670          CS << ",";1671        if (CDS->getElementType()->isIntegerTy()) {1672          APInt Elt = CDS->getElementAsAPInt(i);1673          Elt = IsSext ? Elt.sext(DstEltBits) : Elt.zext(DstEltBits);1674          printConstant(Elt, CS);1675        } else1676          CS << "?";1677      }1678      CS << "]";1679      OutStreamer.AddComment(CS.str());1680      return true;1681    }1682  }1683 1684  return false;1685}1686static void printSignExtend(const MachineInstr *MI, MCStreamer &OutStreamer,1687                            int SrcEltBits, int DstEltBits) {1688  printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, true);1689}1690static void printZeroExtend(const MachineInstr *MI, MCStreamer &OutStreamer,1691                            int SrcEltBits, int DstEltBits) {1692  if (printExtend(MI, OutStreamer, SrcEltBits, DstEltBits, false))1693    return;1694 1695  // We didn't find a constant load, fallback to a shuffle mask decode.1696  std::string Comment;1697  raw_string_ostream CS(Comment);1698  printDstRegisterName(CS, MI, getSrcIdx(MI, 1));1699  CS << " = ";1700 1701  SmallVector<int> Mask;1702  unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);1703  assert((Width % DstEltBits) == 0 && (DstEltBits % SrcEltBits) == 0 &&1704         "Illegal extension ratio");1705  DecodeZeroExtendMask(SrcEltBits, DstEltBits, Width / DstEltBits, false, Mask);1706  printShuffleMask(CS, "mem", "", Mask);1707 1708  OutStreamer.AddComment(CS.str());1709}1710 1711void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {1712  assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");1713  assert((getSubtarget().isOSWindows() || getSubtarget().isUEFI()) &&1714         "SEH_ instruction Windows and UEFI only");1715 1716  // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.1717  if (EmitFPOData) {1718    X86TargetStreamer *XTS =1719        static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer());1720    switch (MI->getOpcode()) {1721    case X86::SEH_PushReg:1722      XTS->emitFPOPushReg(MI->getOperand(0).getImm());1723      break;1724    case X86::SEH_StackAlloc:1725      XTS->emitFPOStackAlloc(MI->getOperand(0).getImm());1726      break;1727    case X86::SEH_StackAlign:1728      XTS->emitFPOStackAlign(MI->getOperand(0).getImm());1729      break;1730    case X86::SEH_SetFrame:1731      assert(MI->getOperand(1).getImm() == 0 &&1732             ".cv_fpo_setframe takes no offset");1733      XTS->emitFPOSetFrame(MI->getOperand(0).getImm());1734      break;1735    case X86::SEH_EndPrologue:1736      XTS->emitFPOEndPrologue();1737      break;1738    case X86::SEH_SaveReg:1739    case X86::SEH_SaveXMM:1740    case X86::SEH_PushFrame:1741      llvm_unreachable("SEH_ directive incompatible with FPO");1742      break;1743    default:1744      llvm_unreachable("expected SEH_ instruction");1745    }1746    return;1747  }1748 1749  // Otherwise, use the .seh_ directives for all other Windows platforms.1750  switch (MI->getOpcode()) {1751  case X86::SEH_PushReg:1752    OutStreamer->emitWinCFIPushReg(MI->getOperand(0).getImm());1753    break;1754 1755  case X86::SEH_SaveReg:1756    OutStreamer->emitWinCFISaveReg(MI->getOperand(0).getImm(),1757                                   MI->getOperand(1).getImm());1758    break;1759 1760  case X86::SEH_SaveXMM:1761    OutStreamer->emitWinCFISaveXMM(MI->getOperand(0).getImm(),1762                                   MI->getOperand(1).getImm());1763    break;1764 1765  case X86::SEH_StackAlloc:1766    OutStreamer->emitWinCFIAllocStack(MI->getOperand(0).getImm());1767    break;1768 1769  case X86::SEH_SetFrame:1770    OutStreamer->emitWinCFISetFrame(MI->getOperand(0).getImm(),1771                                    MI->getOperand(1).getImm());1772    break;1773 1774  case X86::SEH_PushFrame:1775    OutStreamer->emitWinCFIPushFrame(MI->getOperand(0).getImm());1776    break;1777 1778  case X86::SEH_EndPrologue:1779    OutStreamer->emitWinCFIEndProlog();1780    break;1781 1782  case X86::SEH_BeginEpilogue:1783    OutStreamer->emitWinCFIBeginEpilogue();1784    break;1785 1786  case X86::SEH_EndEpilogue:1787    OutStreamer->emitWinCFIEndEpilogue();1788    break;1789 1790  case X86::SEH_UnwindV2Start:1791    OutStreamer->emitWinCFIUnwindV2Start();1792    break;1793 1794  case X86::SEH_UnwindVersion:1795    OutStreamer->emitWinCFIUnwindVersion(MI->getOperand(0).getImm());1796    break;1797 1798  default:1799    llvm_unreachable("expected SEH_ instruction");1800  }1801}1802 1803static void addConstantComments(const MachineInstr *MI,1804                                MCStreamer &OutStreamer) {1805  switch (MI->getOpcode()) {1806  // Lower PSHUFB and VPERMILP normally but add a comment if we can find1807  // a constant shuffle mask. We won't be able to do this at the MC layer1808  // because the mask isn't an immediate.1809  case X86::PSHUFBrm:1810  case X86::VPSHUFBrm:1811  case X86::VPSHUFBYrm:1812  case X86::VPSHUFBZ128rm:1813  case X86::VPSHUFBZ128rmk:1814  case X86::VPSHUFBZ128rmkz:1815  case X86::VPSHUFBZ256rm:1816  case X86::VPSHUFBZ256rmk:1817  case X86::VPSHUFBZ256rmkz:1818  case X86::VPSHUFBZrm:1819  case X86::VPSHUFBZrmk:1820  case X86::VPSHUFBZrmkz: {1821    unsigned SrcIdx = getSrcIdx(MI, 1);1822    if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) {1823      unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);1824      SmallVector<int, 64> Mask;1825      DecodePSHUFBMask(C, Width, Mask);1826      if (!Mask.empty())1827        OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));1828    }1829    break;1830  }1831 1832  case X86::VPERMILPSrm:1833  case X86::VPERMILPSYrm:1834  case X86::VPERMILPSZ128rm:1835  case X86::VPERMILPSZ128rmk:1836  case X86::VPERMILPSZ128rmkz:1837  case X86::VPERMILPSZ256rm:1838  case X86::VPERMILPSZ256rmk:1839  case X86::VPERMILPSZ256rmkz:1840  case X86::VPERMILPSZrm:1841  case X86::VPERMILPSZrmk:1842  case X86::VPERMILPSZrmkz: {1843    unsigned SrcIdx = getSrcIdx(MI, 1);1844    if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) {1845      unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);1846      SmallVector<int, 16> Mask;1847      DecodeVPERMILPMask(C, 32, Width, Mask);1848      if (!Mask.empty())1849        OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));1850    }1851    break;1852  }1853  case X86::VPERMILPDrm:1854  case X86::VPERMILPDYrm:1855  case X86::VPERMILPDZ128rm:1856  case X86::VPERMILPDZ128rmk:1857  case X86::VPERMILPDZ128rmkz:1858  case X86::VPERMILPDZ256rm:1859  case X86::VPERMILPDZ256rmk:1860  case X86::VPERMILPDZ256rmkz:1861  case X86::VPERMILPDZrm:1862  case X86::VPERMILPDZrmk:1863  case X86::VPERMILPDZrmkz: {1864    unsigned SrcIdx = getSrcIdx(MI, 1);1865    if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) {1866      unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);1867      SmallVector<int, 16> Mask;1868      DecodeVPERMILPMask(C, 64, Width, Mask);1869      if (!Mask.empty())1870        OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));1871    }1872    break;1873  }1874 1875  case X86::VPERMIL2PDrm:1876  case X86::VPERMIL2PSrm:1877  case X86::VPERMIL2PDYrm:1878  case X86::VPERMIL2PSYrm: {1879    assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) &&1880           "Unexpected number of operands!");1881 1882    const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1);1883    if (!CtrlOp.isImm())1884      break;1885 1886    unsigned ElSize;1887    switch (MI->getOpcode()) {1888    default: llvm_unreachable("Invalid opcode");1889    case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break;1890    case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break;1891    }1892 1893    if (auto *C = X86::getConstantFromPool(*MI, 3)) {1894      unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);1895      SmallVector<int, 16> Mask;1896      DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Width, Mask);1897      if (!Mask.empty())1898        OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));1899    }1900    break;1901  }1902 1903  case X86::VPPERMrrm: {1904    if (auto *C = X86::getConstantFromPool(*MI, 3)) {1905      unsigned Width = X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);1906      SmallVector<int, 16> Mask;1907      DecodeVPPERMMask(C, Width, Mask);1908      if (!Mask.empty())1909        OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));1910    }1911    break;1912  }1913 1914  case X86::MMX_MOVQ64rm: {1915    if (auto *C = X86::getConstantFromPool(*MI, 1)) {1916      std::string Comment;1917      raw_string_ostream CS(Comment);1918      const MachineOperand &DstOp = MI->getOperand(0);1919      CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";1920      if (auto *CF = dyn_cast<ConstantFP>(C)) {1921        CS << "0x" << toString(CF->getValueAPF().bitcastToAPInt(), 16, false);1922        OutStreamer.AddComment(CS.str());1923      }1924    }1925    break;1926  }1927 1928#define INSTR_CASE(Prefix, Instr, Suffix, Postfix)                             \1929  case X86::Prefix##Instr##Suffix##rm##Postfix:1930 1931#define CASE_AVX512_ARITH_RM(Instr)                                            \1932  INSTR_CASE(V, Instr, Z128, )                                                 \1933  INSTR_CASE(V, Instr, Z128, k)                                                \1934  INSTR_CASE(V, Instr, Z128, kz)                                               \1935  INSTR_CASE(V, Instr, Z256, )                                                 \1936  INSTR_CASE(V, Instr, Z256, k)                                                \1937  INSTR_CASE(V, Instr, Z256, kz)                                               \1938  INSTR_CASE(V, Instr, Z, )                                                    \1939  INSTR_CASE(V, Instr, Z, k)                                                   \1940  INSTR_CASE(V, Instr, Z, kz)1941 1942#define CASE_ARITH_RM(Instr)                                                   \1943  INSTR_CASE(, Instr, , )   /* SSE */                                          \1944  INSTR_CASE(V, Instr, , )  /* AVX-128 */                                      \1945  INSTR_CASE(V, Instr, Y, ) /* AVX-256 */                                      \1946  INSTR_CASE(V, Instr, Z128, )                                                 \1947  INSTR_CASE(V, Instr, Z128, k)                                                \1948  INSTR_CASE(V, Instr, Z128, kz)                                               \1949  INSTR_CASE(V, Instr, Z256, )                                                 \1950  INSTR_CASE(V, Instr, Z256, k)                                                \1951  INSTR_CASE(V, Instr, Z256, kz)                                               \1952  INSTR_CASE(V, Instr, Z, )                                                    \1953  INSTR_CASE(V, Instr, Z, k)                                                   \1954  INSTR_CASE(V, Instr, Z, kz)1955 1956    // TODO: Add additional instructions when useful.1957    CASE_ARITH_RM(PMADDUBSW)1958    CASE_ARITH_RM(PMADDWD)1959    CASE_ARITH_RM(PMULDQ)1960    CASE_ARITH_RM(PMULUDQ)1961    CASE_ARITH_RM(PMULLD)1962    CASE_AVX512_ARITH_RM(PMULLQ)1963    CASE_ARITH_RM(PMULLW)1964    CASE_ARITH_RM(PMULHW)1965    CASE_ARITH_RM(PMULHUW)1966    CASE_ARITH_RM(PMULHRSW) {1967      unsigned SrcIdx = getSrcIdx(MI, 1);1968      if (auto *C = X86::getConstantFromPool(*MI, SrcIdx + 1)) {1969        std::string Comment;1970        raw_string_ostream CS(Comment);1971        unsigned VectorWidth =1972            X86::getVectorRegisterWidth(MI->getDesc().operands()[0]);1973        CS << "[";1974        printConstant(C, VectorWidth, CS);1975        CS << "]";1976        OutStreamer.AddComment(CS.str());1977      }1978      break;1979    }1980 1981#define MASK_AVX512_CASE(Instr)                                                \1982  case Instr:                                                                  \1983  case Instr##k:                                                               \1984  case Instr##kz:1985 1986  case X86::MOVSDrm:1987  case X86::VMOVSDrm:1988  MASK_AVX512_CASE(X86::VMOVSDZrm)1989  case X86::MOVSDrm_alt:1990  case X86::VMOVSDrm_alt:1991  case X86::VMOVSDZrm_alt:1992  case X86::MOVQI2PQIrm:1993  case X86::VMOVQI2PQIrm:1994  case X86::VMOVQI2PQIZrm:1995    printZeroUpperMove(MI, OutStreamer, 64, 128, "mem[0],zero");1996    break;1997 1998  MASK_AVX512_CASE(X86::VMOVSHZrm)1999  case X86::VMOVSHZrm_alt:2000    printZeroUpperMove(MI, OutStreamer, 16, 128,2001                       "mem[0],zero,zero,zero,zero,zero,zero,zero");2002    break;2003 2004  case X86::MOVSSrm:2005  case X86::VMOVSSrm:2006  MASK_AVX512_CASE(X86::VMOVSSZrm)2007  case X86::MOVSSrm_alt:2008  case X86::VMOVSSrm_alt:2009  case X86::VMOVSSZrm_alt:2010  case X86::MOVDI2PDIrm:2011  case X86::VMOVDI2PDIrm:2012  case X86::VMOVDI2PDIZrm:2013    printZeroUpperMove(MI, OutStreamer, 32, 128, "mem[0],zero,zero,zero");2014    break;2015 2016#define MOV_CASE(Prefix, Suffix)                                               \2017  case X86::Prefix##MOVAPD##Suffix##rm:                                        \2018  case X86::Prefix##MOVAPS##Suffix##rm:                                        \2019  case X86::Prefix##MOVUPD##Suffix##rm:                                        \2020  case X86::Prefix##MOVUPS##Suffix##rm:                                        \2021  case X86::Prefix##MOVDQA##Suffix##rm:                                        \2022  case X86::Prefix##MOVDQU##Suffix##rm:2023 2024#define MOV_AVX512_CASE(Suffix, Postfix)                                       \2025  case X86::VMOVDQA64##Suffix##rm##Postfix:                                    \2026  case X86::VMOVDQA32##Suffix##rm##Postfix:                                    \2027  case X86::VMOVDQU64##Suffix##rm##Postfix:                                    \2028  case X86::VMOVDQU32##Suffix##rm##Postfix:                                    \2029  case X86::VMOVDQU16##Suffix##rm##Postfix:                                    \2030  case X86::VMOVDQU8##Suffix##rm##Postfix:                                     \2031  case X86::VMOVAPS##Suffix##rm##Postfix:                                      \2032  case X86::VMOVAPD##Suffix##rm##Postfix:                                      \2033  case X86::VMOVUPS##Suffix##rm##Postfix:                                      \2034  case X86::VMOVUPD##Suffix##rm##Postfix:2035 2036#define CASE_128_MOV_RM()                                                      \2037  MOV_CASE(, )   /* SSE */                                                     \2038  MOV_CASE(V, )  /* AVX-128 */                                                 \2039  MOV_AVX512_CASE(Z128, )                                                      \2040  MOV_AVX512_CASE(Z128, k)                                                     \2041  MOV_AVX512_CASE(Z128, kz)2042 2043#define CASE_256_MOV_RM()                                                      \2044  MOV_CASE(V, Y) /* AVX-256 */                                                 \2045  MOV_AVX512_CASE(Z256, )                                                      \2046  MOV_AVX512_CASE(Z256, k)                                                     \2047  MOV_AVX512_CASE(Z256, kz)                                                    \2048 2049#define CASE_512_MOV_RM()                                                      \2050  MOV_AVX512_CASE(Z, )                                                         \2051  MOV_AVX512_CASE(Z, k)                                                        \2052  MOV_AVX512_CASE(Z, kz)                                                       \2053 2054    // For loads from a constant pool to a vector register, print the constant2055    // loaded.2056    CASE_128_MOV_RM()2057    printBroadcast(MI, OutStreamer, 1, 128);2058    break;2059    CASE_256_MOV_RM()2060    printBroadcast(MI, OutStreamer, 1, 256);2061    break;2062    CASE_512_MOV_RM()2063    printBroadcast(MI, OutStreamer, 1, 512);2064    break;2065  case X86::VBROADCASTF128rm:2066  case X86::VBROADCASTI128rm:2067  MASK_AVX512_CASE(X86::VBROADCASTF32X4Z256rm)2068  MASK_AVX512_CASE(X86::VBROADCASTF64X2Z256rm)2069  MASK_AVX512_CASE(X86::VBROADCASTI32X4Z256rm)2070  MASK_AVX512_CASE(X86::VBROADCASTI64X2Z256rm)2071    printBroadcast(MI, OutStreamer, 2, 128);2072    break;2073  MASK_AVX512_CASE(X86::VBROADCASTF32X4Zrm)2074  MASK_AVX512_CASE(X86::VBROADCASTF64X2Zrm)2075  MASK_AVX512_CASE(X86::VBROADCASTI32X4Zrm)2076  MASK_AVX512_CASE(X86::VBROADCASTI64X2Zrm)2077    printBroadcast(MI, OutStreamer, 4, 128);2078    break;2079  MASK_AVX512_CASE(X86::VBROADCASTF32X8Zrm)2080  MASK_AVX512_CASE(X86::VBROADCASTF64X4Zrm)2081  MASK_AVX512_CASE(X86::VBROADCASTI32X8Zrm)2082  MASK_AVX512_CASE(X86::VBROADCASTI64X4Zrm)2083    printBroadcast(MI, OutStreamer, 2, 256);2084    break;2085 2086  // For broadcast loads from a constant pool to a vector register, repeatedly2087  // print the constant loaded.2088  case X86::MOVDDUPrm:2089  case X86::VMOVDDUPrm:2090  MASK_AVX512_CASE(X86::VMOVDDUPZ128rm)2091  case X86::VPBROADCASTQrm:2092  MASK_AVX512_CASE(X86::VPBROADCASTQZ128rm)2093    printBroadcast(MI, OutStreamer, 2, 64);2094    break;2095  case X86::VBROADCASTSDYrm:2096  MASK_AVX512_CASE(X86::VBROADCASTSDZ256rm)2097  case X86::VPBROADCASTQYrm:2098  MASK_AVX512_CASE(X86::VPBROADCASTQZ256rm)2099    printBroadcast(MI, OutStreamer, 4, 64);2100    break;2101  MASK_AVX512_CASE(X86::VBROADCASTSDZrm)2102  MASK_AVX512_CASE(X86::VPBROADCASTQZrm)2103    printBroadcast(MI, OutStreamer, 8, 64);2104    break;2105  case X86::VBROADCASTSSrm:2106  MASK_AVX512_CASE(X86::VBROADCASTSSZ128rm)2107  case X86::VPBROADCASTDrm:2108  MASK_AVX512_CASE(X86::VPBROADCASTDZ128rm)2109    printBroadcast(MI, OutStreamer, 4, 32);2110    break;2111  case X86::VBROADCASTSSYrm:2112    MASK_AVX512_CASE(X86::VBROADCASTSSZ256rm)2113  case X86::VPBROADCASTDYrm:2114  MASK_AVX512_CASE(X86::VPBROADCASTDZ256rm)2115    printBroadcast(MI, OutStreamer, 8, 32);2116    break;2117  MASK_AVX512_CASE(X86::VBROADCASTSSZrm)2118  MASK_AVX512_CASE(X86::VPBROADCASTDZrm)2119    printBroadcast(MI, OutStreamer, 16, 32);2120    break;2121  case X86::VPBROADCASTWrm:2122  MASK_AVX512_CASE(X86::VPBROADCASTWZ128rm)2123    printBroadcast(MI, OutStreamer, 8, 16);2124    break;2125  case X86::VPBROADCASTWYrm:2126  MASK_AVX512_CASE(X86::VPBROADCASTWZ256rm)2127    printBroadcast(MI, OutStreamer, 16, 16);2128    break;2129  MASK_AVX512_CASE(X86::VPBROADCASTWZrm)2130    printBroadcast(MI, OutStreamer, 32, 16);2131    break;2132  case X86::VPBROADCASTBrm:2133  MASK_AVX512_CASE(X86::VPBROADCASTBZ128rm)2134    printBroadcast(MI, OutStreamer, 16, 8);2135    break;2136  case X86::VPBROADCASTBYrm:2137  MASK_AVX512_CASE(X86::VPBROADCASTBZ256rm)2138    printBroadcast(MI, OutStreamer, 32, 8);2139    break;2140  MASK_AVX512_CASE(X86::VPBROADCASTBZrm)2141    printBroadcast(MI, OutStreamer, 64, 8);2142    break;2143 2144#define MOVX_CASE(Prefix, Ext, Type, Suffix, Postfix)                          \2145  case X86::Prefix##PMOV##Ext##Type##Suffix##rm##Postfix:2146 2147#define CASE_MOVX_RM(Ext, Type)                                                \2148  MOVX_CASE(, Ext, Type, , )                                                   \2149  MOVX_CASE(V, Ext, Type, , )                                                  \2150  MOVX_CASE(V, Ext, Type, Y, )                                                 \2151  MOVX_CASE(V, Ext, Type, Z128, )                                              \2152  MOVX_CASE(V, Ext, Type, Z128, k )                                            \2153  MOVX_CASE(V, Ext, Type, Z128, kz )                                           \2154  MOVX_CASE(V, Ext, Type, Z256, )                                              \2155  MOVX_CASE(V, Ext, Type, Z256, k )                                            \2156  MOVX_CASE(V, Ext, Type, Z256, kz )                                           \2157  MOVX_CASE(V, Ext, Type, Z, )                                                 \2158  MOVX_CASE(V, Ext, Type, Z, k )                                               \2159  MOVX_CASE(V, Ext, Type, Z, kz )2160 2161    CASE_MOVX_RM(SX, BD)2162    printSignExtend(MI, OutStreamer, 8, 32);2163    break;2164    CASE_MOVX_RM(SX, BQ)2165    printSignExtend(MI, OutStreamer, 8, 64);2166    break;2167    CASE_MOVX_RM(SX, BW)2168    printSignExtend(MI, OutStreamer, 8, 16);2169    break;2170    CASE_MOVX_RM(SX, DQ)2171    printSignExtend(MI, OutStreamer, 32, 64);2172    break;2173    CASE_MOVX_RM(SX, WD)2174    printSignExtend(MI, OutStreamer, 16, 32);2175    break;2176    CASE_MOVX_RM(SX, WQ)2177    printSignExtend(MI, OutStreamer, 16, 64);2178    break;2179 2180    CASE_MOVX_RM(ZX, BD)2181    printZeroExtend(MI, OutStreamer, 8, 32);2182    break;2183    CASE_MOVX_RM(ZX, BQ)2184    printZeroExtend(MI, OutStreamer, 8, 64);2185    break;2186    CASE_MOVX_RM(ZX, BW)2187    printZeroExtend(MI, OutStreamer, 8, 16);2188    break;2189    CASE_MOVX_RM(ZX, DQ)2190    printZeroExtend(MI, OutStreamer, 32, 64);2191    break;2192    CASE_MOVX_RM(ZX, WD)2193    printZeroExtend(MI, OutStreamer, 16, 32);2194    break;2195    CASE_MOVX_RM(ZX, WQ)2196    printZeroExtend(MI, OutStreamer, 16, 64);2197    break;2198  }2199}2200 2201// Does the given operand refer to a DLLIMPORT function?2202bool isImportedFunction(const MachineOperand &MO) {2203  return MO.isGlobal() && (MO.getTargetFlags() == X86II::MO_DLLIMPORT);2204}2205 2206// Is the given instruction a call to a CFGuard function?2207bool isCallToCFGuardFunction(const MachineInstr *MI) {2208  assert(MI->getOpcode() == X86::TAILJMPm64_REX ||2209         MI->getOpcode() == X86::CALL64m);2210  const MachineOperand &MO = MI->getOperand(3);2211  return MO.isGlobal() && (MO.getTargetFlags() == X86II::MO_NO_FLAG) &&2212         isCFGuardFunction(MO.getGlobal());2213}2214 2215// Does the containing block for the given instruction contain any jump table2216// info (indicating that the block is a dispatch for a jump table)?2217bool hasJumpTableInfoInBlock(const llvm::MachineInstr *MI) {2218  const MachineBasicBlock &MBB = *MI->getParent();2219  for (auto I = MBB.instr_rbegin(), E = MBB.instr_rend(); I != E; ++I)2220    if (I->isJumpTableDebugInfo())2221      return true;2222 2223  return false;2224}2225 2226void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {2227  // FIXME: Enable feature predicate checks once all the test pass.2228  // X86_MC::verifyInstructionPredicates(MI->getOpcode(),2229  //                                     Subtarget->getFeatureBits());2230 2231  X86MCInstLower MCInstLowering(*MF, *this);2232  const X86RegisterInfo *RI =2233      MF->getSubtarget<X86Subtarget>().getRegisterInfo();2234 2235  if (MI->getOpcode() == X86::OR64rm) {2236    for (auto &Opd : MI->operands()) {2237      if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==2238                                "swift_async_extendedFramePointerFlags") {2239        ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;2240      }2241    }2242  }2243 2244  // Add comments for values loaded from constant pool.2245  if (OutStreamer->isVerboseAsm())2246    addConstantComments(MI, *OutStreamer);2247 2248  // Add a comment about EVEX compression2249  if (TM.Options.MCOptions.ShowMCEncoding) {2250    if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_LEGACY)2251      OutStreamer->AddComment("EVEX TO LEGACY Compression ", false);2252    else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX)2253      OutStreamer->AddComment("EVEX TO VEX Compression ", false);2254    else if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_EVEX)2255      OutStreamer->AddComment("EVEX TO EVEX Compression ", false);2256  }2257 2258  // We use this to suppress NOP padding for Windows EH.2259  bool IsTailJump = false;2260 2261  switch (MI->getOpcode()) {2262  case TargetOpcode::DBG_VALUE:2263    llvm_unreachable("Should be handled target independently");2264 2265  case X86::EH_RETURN:2266  case X86::EH_RETURN64: {2267    // Lower these as normal, but add some comments.2268    Register Reg = MI->getOperand(0).getReg();2269    OutStreamer->AddComment(StringRef("eh_return, addr: %") +2270                            X86ATTInstPrinter::getRegisterName(Reg));2271    break;2272  }2273  case X86::CLEANUPRET: {2274    // Lower these as normal, but add some comments.2275    OutStreamer->AddComment("CLEANUPRET");2276    break;2277  }2278 2279  case X86::CATCHRET: {2280    // Lower these as normal, but add some comments.2281    OutStreamer->AddComment("CATCHRET");2282    break;2283  }2284 2285  case X86::ENDBR32:2286  case X86::ENDBR64: {2287    // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for2288    // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be2289    // non-empty. If MI is the initial ENDBR, place the2290    // __patchable_function_entries label after ENDBR.2291    if (CurrentPatchableFunctionEntrySym &&2292        CurrentPatchableFunctionEntrySym == CurrentFnBegin &&2293        MI == &MF->front().front()) {2294      MCInst Inst;2295      MCInstLowering.Lower(MI, Inst);2296      EmitAndCountInstruction(Inst);2297      CurrentPatchableFunctionEntrySym = createTempSymbol("patch");2298      OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);2299      return;2300    }2301    break;2302  }2303 2304  case X86::TAILJMPd64:2305    if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(X86::R11))2306      EmitAndCountInstruction(MCInstBuilder(X86::CS_PREFIX));2307 2308    if (EnableImportCallOptimization && isImportedFunction(MI->getOperand(0))) {2309      emitLabelAndRecordForImportCallOptimization(2310          IMAGE_RETPOLINE_AMD64_IMPORT_BR);2311    }2312 2313    // Lower this as normal, but add a comment.2314    OutStreamer->AddComment("TAILCALL");2315    IsTailJump = true;2316    break;2317 2318  case X86::TAILJMPr:2319  case X86::TAILJMPm:2320  case X86::TAILJMPd:2321  case X86::TAILJMPd_CC:2322  case X86::TAILJMPr64:2323  case X86::TAILJMPm64:2324  case X86::TAILJMPd64_CC:2325    if (EnableImportCallOptimization)2326      report_fatal_error("Unexpected TAILJMP instruction was emitted when "2327                         "import call optimization was enabled");2328 2329    // Lower these as normal, but add some comments.2330    OutStreamer->AddComment("TAILCALL");2331    IsTailJump = true;2332    break;2333 2334  case X86::TAILJMPm64_REX:2335    if (EnableImportCallOptimization && isCallToCFGuardFunction(MI)) {2336      emitLabelAndRecordForImportCallOptimization(2337          IMAGE_RETPOLINE_AMD64_CFG_BR_REX);2338    }2339 2340    OutStreamer->AddComment("TAILCALL");2341    IsTailJump = true;2342    break;2343 2344  case X86::TAILJMPr64_REX: {2345    if (EnableImportCallOptimization) {2346      assert(MI->getOperand(0).getReg() == X86::RAX &&2347             "Indirect tail calls with impcall enabled must go through RAX (as "2348             "enforced by TCRETURNImpCallri64)");2349      emitLabelAndRecordForImportCallOptimization(2350          IMAGE_RETPOLINE_AMD64_INDIR_BR);2351    }2352 2353    OutStreamer->AddComment("TAILCALL");2354    IsTailJump = true;2355    break;2356  }2357 2358  case X86::JMP64r:2359    if (EnableImportCallOptimization && hasJumpTableInfoInBlock(MI)) {2360      uint16_t EncodedReg =2361          this->getSubtarget().getRegisterInfo()->getEncodingValue(2362              MI->getOperand(0).getReg().asMCReg());2363      emitLabelAndRecordForImportCallOptimization(2364          (ImportCallKind)(IMAGE_RETPOLINE_AMD64_SWITCHTABLE_FIRST +2365                           EncodedReg));2366    }2367    break;2368 2369  case X86::JMP16r:2370  case X86::JMP16m:2371  case X86::JMP32r:2372  case X86::JMP32m:2373  case X86::JMP64m:2374    if (EnableImportCallOptimization && hasJumpTableInfoInBlock(MI))2375      report_fatal_error(2376          "Unexpected JMP instruction was emitted for a jump-table when import "2377          "call optimization was enabled");2378    break;2379 2380  case X86::TLS_addr32:2381  case X86::TLS_addr64:2382  case X86::TLS_addrX32:2383  case X86::TLS_base_addr32:2384  case X86::TLS_base_addr64:2385  case X86::TLS_base_addrX32:2386  case X86::TLS_desc32:2387  case X86::TLS_desc64:2388    return LowerTlsAddr(MCInstLowering, *MI);2389 2390  case X86::MOVPC32r: {2391    // This is a pseudo op for a two instruction sequence with a label, which2392    // looks like:2393    //     call "L1$pb"2394    // "L1$pb":2395    //     popl %esi2396 2397    // Emit the call.2398    MCSymbol *PICBase = MF->getPICBaseSymbol();2399    // FIXME: We would like an efficient form for this, so we don't have to do a2400    // lot of extra uniquing.2401    EmitAndCountInstruction(2402        MCInstBuilder(X86::CALLpcrel32)2403            .addExpr(MCSymbolRefExpr::create(PICBase, OutContext)));2404 2405    const X86FrameLowering *FrameLowering =2406        MF->getSubtarget<X86Subtarget>().getFrameLowering();2407    bool hasFP = FrameLowering->hasFP(*MF);2408 2409    // TODO: This is needed only if we require precise CFA.2410    bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() &&2411                               !OutStreamer->getDwarfFrameInfos().back().End;2412 2413    int stackGrowth = -RI->getSlotSize();2414 2415    if (HasActiveDwarfFrame && !hasFP) {2416      OutStreamer->emitCFIAdjustCfaOffset(-stackGrowth);2417      MF->getInfo<X86MachineFunctionInfo>()->setHasCFIAdjustCfa(true);2418    }2419 2420    // Emit the label.2421    OutStreamer->emitLabel(PICBase);2422 2423    // popl $reg2424    EmitAndCountInstruction(2425        MCInstBuilder(X86::POP32r).addReg(MI->getOperand(0).getReg()));2426 2427    if (HasActiveDwarfFrame && !hasFP) {2428      OutStreamer->emitCFIAdjustCfaOffset(stackGrowth);2429    }2430    return;2431  }2432 2433  case X86::ADD32ri: {2434    // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri.2435    if (MI->getOperand(2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS)2436      break;2437 2438    // Okay, we have something like:2439    //  EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL)2440 2441    // For this, we want to print something like:2442    //   MYGLOBAL + (. - PICBASE)2443    // However, we can't generate a ".", so just emit a new label here and refer2444    // to it.2445    MCSymbol *DotSym = OutContext.createTempSymbol();2446    OutStreamer->emitLabel(DotSym);2447 2448    // Now that we have emitted the label, lower the complex operand expression.2449    MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2));2450 2451    const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext);2452    const MCExpr *PICBase =2453        MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext);2454    DotExpr = MCBinaryExpr::createSub(DotExpr, PICBase, OutContext);2455 2456    DotExpr = MCBinaryExpr::createAdd(2457        MCSymbolRefExpr::create(OpSym, OutContext), DotExpr, OutContext);2458 2459    EmitAndCountInstruction(MCInstBuilder(X86::ADD32ri)2460                                .addReg(MI->getOperand(0).getReg())2461                                .addReg(MI->getOperand(1).getReg())2462                                .addExpr(DotExpr));2463    return;2464  }2465  case TargetOpcode::STATEPOINT:2466    return LowerSTATEPOINT(*MI, MCInstLowering);2467 2468  case TargetOpcode::FAULTING_OP:2469    return LowerFAULTING_OP(*MI, MCInstLowering);2470 2471  case TargetOpcode::FENTRY_CALL:2472    return LowerFENTRY_CALL(*MI, MCInstLowering);2473 2474  case TargetOpcode::PATCHABLE_OP:2475    return LowerPATCHABLE_OP(*MI, MCInstLowering);2476 2477  case TargetOpcode::STACKMAP:2478    return LowerSTACKMAP(*MI);2479 2480  case TargetOpcode::PATCHPOINT:2481    return LowerPATCHPOINT(*MI, MCInstLowering);2482 2483  case TargetOpcode::PATCHABLE_FUNCTION_ENTER:2484    return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering);2485 2486  case TargetOpcode::PATCHABLE_RET:2487    return LowerPATCHABLE_RET(*MI, MCInstLowering);2488 2489  case TargetOpcode::PATCHABLE_TAIL_CALL:2490    return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering);2491 2492  case TargetOpcode::PATCHABLE_EVENT_CALL:2493    return LowerPATCHABLE_EVENT_CALL(*MI, MCInstLowering);2494 2495  case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:2496    return LowerPATCHABLE_TYPED_EVENT_CALL(*MI, MCInstLowering);2497 2498  case X86::MORESTACK_RET:2499    EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));2500    return;2501 2502  case X86::KCFI_CHECK:2503    return LowerKCFI_CHECK(*MI);2504 2505  case X86::ASAN_CHECK_MEMACCESS:2506    return LowerASAN_CHECK_MEMACCESS(*MI);2507 2508  case X86::MORESTACK_RET_RESTORE_R10:2509    // Return, then restore R10.2510    EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));2511    EmitAndCountInstruction(2512        MCInstBuilder(X86::MOV64rr).addReg(X86::R10).addReg(X86::RAX));2513    return;2514 2515  case X86::SEH_PushReg:2516  case X86::SEH_SaveReg:2517  case X86::SEH_SaveXMM:2518  case X86::SEH_StackAlloc:2519  case X86::SEH_StackAlign:2520  case X86::SEH_SetFrame:2521  case X86::SEH_PushFrame:2522  case X86::SEH_EndPrologue:2523  case X86::SEH_EndEpilogue:2524  case X86::SEH_UnwindV2Start:2525  case X86::SEH_UnwindVersion:2526    EmitSEHInstruction(MI);2527    return;2528 2529  case X86::SEH_BeginEpilogue: {2530    assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");2531    EmitSEHInstruction(MI);2532    return;2533  }2534  case X86::UBSAN_UD1:2535    EmitAndCountInstruction(MCInstBuilder(X86::UD1Lm)2536                                .addReg(X86::EAX)2537                                .addReg(X86::EAX)2538                                .addImm(1)2539                                .addReg(X86::NoRegister)2540                                .addImm(MI->getOperand(0).getImm())2541                                .addReg(X86::NoRegister));2542    return;2543  case X86::CALL64pcrel32:2544    if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(X86::R11))2545      EmitAndCountInstruction(MCInstBuilder(X86::CS_PREFIX));2546 2547    if (EnableImportCallOptimization && isImportedFunction(MI->getOperand(0))) {2548      emitLabelAndRecordForImportCallOptimization(2549          IMAGE_RETPOLINE_AMD64_IMPORT_CALL);2550 2551      MCInst TmpInst;2552      MCInstLowering.Lower(MI, TmpInst);2553 2554      // For Import Call Optimization to work, we need a the call instruction2555      // with a rex prefix, and a 5-byte nop after the call instruction.2556      EmitAndCountInstruction(MCInstBuilder(X86::REX64_PREFIX));2557      emitCallInstruction(TmpInst);2558      emitNop(*OutStreamer, 5, Subtarget);2559      maybeEmitNopAfterCallForWindowsEH(MI);2560      return;2561    }2562 2563    break;2564 2565  case X86::CALL64r:2566    if (EnableImportCallOptimization) {2567      assert(MI->getOperand(0).getReg() == X86::RAX &&2568             "Indirect calls with impcall enabled must go through RAX (as "2569             "enforced by CALL64r_ImpCall)");2570 2571      emitLabelAndRecordForImportCallOptimization(2572          IMAGE_RETPOLINE_AMD64_INDIR_CALL);2573      MCInst TmpInst;2574      MCInstLowering.Lower(MI, TmpInst);2575      emitCallInstruction(TmpInst);2576 2577      // For Import Call Optimization to work, we need a 3-byte nop after the2578      // call instruction.2579      emitNop(*OutStreamer, 3, Subtarget);2580      maybeEmitNopAfterCallForWindowsEH(MI);2581      return;2582    }2583    break;2584 2585  case X86::CALL64m:2586    if (EnableImportCallOptimization && isCallToCFGuardFunction(MI)) {2587      emitLabelAndRecordForImportCallOptimization(2588          IMAGE_RETPOLINE_AMD64_CFG_CALL);2589    }2590    break;2591 2592  case X86::JCC_1:2593    // Two instruction prefixes (2EH for branch not-taken and 3EH for branch2594    // taken) are used as branch hints. Here we add branch taken prefix for2595    // jump instruction with higher probability than threshold.2596    if (getSubtarget().hasBranchHint() && EnableBranchHint) {2597      const MachineBranchProbabilityInfo *MBPI =2598          &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();2599      MachineBasicBlock *DestBB = MI->getOperand(0).getMBB();2600      BranchProbability EdgeProb =2601          MBPI->getEdgeProbability(MI->getParent(), DestBB);2602      BranchProbability Threshold(BranchHintProbabilityThreshold, 100);2603      if (EdgeProb > Threshold)2604        EmitAndCountInstruction(MCInstBuilder(X86::DS_PREFIX));2605    }2606    break;2607  }2608 2609  MCInst TmpInst;2610  MCInstLowering.Lower(MI, TmpInst);2611 2612  if (MI->isCall()) {2613    emitCallInstruction(TmpInst);2614    // Since tail calls transfer control without leaving a stack frame, there is2615    // never a need for NOP padding tail calls.2616    if (!IsTailJump)2617      maybeEmitNopAfterCallForWindowsEH(MI);2618    return;2619  }2620 2621  EmitAndCountInstruction(TmpInst);2622}2623 2624void X86AsmPrinter::emitCallInstruction(const llvm::MCInst &MCI) {2625  // Stackmap shadows cannot include branch targets, so we can count the bytes2626  // in a call towards the shadow, but must ensure that the no thread returns2627  // in to the stackmap shadow.  The only way to achieve this is if the call2628  // is at the end of the shadow.2629 2630  // Count then size of the call towards the shadow2631  SMShadowTracker.count(MCI, getSubtargetInfo(), CodeEmitter.get());2632  // Then flush the shadow so that we fill with nops before the call, not2633  // after it.2634  SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());2635  // Then emit the call2636  OutStreamer->emitInstruction(MCI, getSubtargetInfo());2637}2638 2639// Determines whether a NOP is required after a CALL, so that Windows EH2640// IP2State tables have the correct information.2641//2642// On most Windows platforms (AMD64, ARM64, ARM32, IA64, but *not* x86-32),2643// exception handling works by looking up instruction pointers in lookup2644// tables. These lookup tables are stored in .xdata sections in executables.2645// One element of the lookup tables are the "IP2State" tables (Instruction2646// Pointer to State).2647//2648// If a function has any instructions that require cleanup during exception2649// unwinding, then it will have an IP2State table. Each entry in the IP2State2650// table describes a range of bytes in the function's instruction stream, and2651// associates an "EH state number" with that range of instructions. A value of2652// -1 means "the null state", which does not require any code to execute.2653// A value other than -1 is an index into the State table.2654//2655// The entries in the IP2State table contain byte offsets within the instruction2656// stream of the function. The Windows ABI requires that these offsets are2657// aligned to instruction boundaries; they are not permitted to point to a byte2658// that is not the first byte of an instruction.2659//2660// Unfortunately, CALL instructions present a problem during unwinding. CALL2661// instructions push the address of the instruction after the CALL instruction,2662// so that execution can resume after the CALL. If the CALL is the last2663// instruction within an IP2State region, then the return address (on the stack)2664// points to the *next* IP2State region. This means that the unwinder will2665// use the wrong cleanup funclet during unwinding.2666//2667// To fix this problem, the Windows AMD64 ABI requires that CALL instructions2668// are never placed at the end of an IP2State region. Stated equivalently, the2669// end of a CALL instruction cannot be aligned to an IP2State boundary.  If a2670// CALL instruction would occur at the end of an IP2State region, then the2671// compiler must insert a NOP instruction after the CALL. The NOP instruction2672// is placed in the same EH region as the CALL instruction, so that the return2673// address points to the NOP and the unwinder will locate the correct region.2674//2675// NOP padding is only necessary on Windows AMD64 targets. On ARM64 and ARM32,2676// instructions have a fixed size so the unwinder knows how to "back up" by2677// one instruction.2678//2679// Interaction with Import Call Optimization (ICO):2680//2681// Import Call Optimization (ICO) is a compiler + OS feature on Windows which2682// improves the performance and security of DLL imports. ICO relies on using a2683// specific CALL idiom that can be replaced by the OS DLL loader. This removes2684// a load and indirect CALL and replaces it with a single direct CALL.2685//2686// To achieve this, ICO also inserts NOPs after the CALL instruction. If the2687// end of the CALL is aligned with an EH state transition, we *also* insert2688// a single-byte NOP.  **Both forms of NOPs must be preserved.**  They cannot2689// be combined into a single larger NOP; nor can the second NOP be removed.2690//2691// This is necessary because, if ICO is active and the call site is modified2692// by the loader, the loader will end up overwriting the NOPs that were inserted2693// for ICO. That means that those NOPs cannot be used for the correct2694// termination of the exception handling region (the IP2State transition),2695// so we still need an additional NOP instruction.  The NOPs cannot be combined2696// into a longer NOP (which is ordinarily desirable) because then ICO would2697// split one instruction, producing a malformed instruction after the ICO call.2698void X86AsmPrinter::maybeEmitNopAfterCallForWindowsEH(const MachineInstr *MI) {2699  // We only need to insert NOPs after CALLs when targeting Windows on AMD64.2700  // (Don't let the name fool you: Itanium refers to table-based exception2701  // handling, not the Itanium architecture.)2702  if (MAI->getExceptionHandlingType() != ExceptionHandling::WinEH ||2703      MAI->getWinEHEncodingType() != WinEH::EncodingType::Itanium) {2704    return;2705  }2706 2707  bool HasEHPersonality = MF->getWinEHFuncInfo() != nullptr;2708 2709  // Set up MBB iterator, initially positioned on the same MBB as MI.2710  MachineFunction::const_iterator MFI(MI->getParent());2711  MachineFunction::const_iterator MFE(MF->end());2712 2713  // Set up instruction iterator, positioned immediately *after* MI.2714  MachineBasicBlock::const_iterator MBBI(MI);2715  MachineBasicBlock::const_iterator MBBE = MI->getParent()->end();2716  ++MBBI; // Step over MI2717 2718  // This loop iterates MBBs2719  for (;;) {2720    // This loop iterates instructions2721    for (; MBBI != MBBE; ++MBBI) {2722      // Check the instruction that follows this CALL.2723      const MachineInstr &NextMI = *MBBI;2724 2725      // If there is an EH_LABEL after this CALL, then there is an EH state2726      // transition after this CALL. This is exactly the situation which2727      // requires NOP padding.2728      if (NextMI.isEHLabel()) {2729        if (HasEHPersonality) {2730          EmitAndCountInstruction(MCInstBuilder(X86::NOOP));2731          return;2732        }2733        // We actually want to continue, in case there is an SEH_BeginEpilogue2734        // instruction after the EH_LABEL. In some situations, IR is produced2735        // that contains EH_LABEL pseudo-instructions, even when we are not2736        // generating IP2State tables. We still need to insert a NOP before2737        // SEH_BeginEpilogue in that case.2738        continue;2739      }2740 2741      // Somewhat similarly, if the CALL is the last instruction before the2742      // SEH prologue, then we also need a NOP. This is necessary because the2743      // Windows stack unwinder will not invoke a function's exception handler2744      // if the instruction pointer is in the function prologue or epilogue.2745      //2746      // We always emit a NOP before SEH_BeginEpilogue, even if there is no2747      // personality function (unwind info) for this frame. This is the same2748      // behavior as MSVC.2749      if (NextMI.getOpcode() == X86::SEH_BeginEpilogue) {2750        EmitAndCountInstruction(MCInstBuilder(X86::NOOP));2751        return;2752      }2753 2754      if (!NextMI.isPseudo() && !NextMI.isMetaInstruction()) {2755        // We found a real instruction. During the CALL, the return IP will2756        // point to this instruction. Since this instruction has the same EH2757        // state as the call itself (because there is no intervening EH_LABEL),2758        // the IP2State table will be accurate; there is no need to insert a2759        // NOP.2760        return;2761      }2762 2763      // The next instruction is a pseudo-op. Ignore it and keep searching.2764      // Because these instructions do not generate any machine code, they2765      // cannot prevent the IP2State table from pointing at the wrong2766      // instruction during a CALL.2767    }2768 2769    // We've reached the end of this MBB. Find the next MBB in program order.2770    // MBB order should be finalized by this point, so falling across MBBs is2771    // expected.2772    ++MFI;2773    if (MFI == MFE) {2774      // No more blocks; we've reached the end of the function. This should2775      // only happen with no-return functions, but double-check to be sure.2776      if (HasEHPersonality) {2777        // If the CALL has no successors, then it is a noreturn function.2778        // Insert an INT3 instead of a NOP. This accomplishes the same purpose,2779        // but is more clear to read. Also, analysis tools will understand2780        // that they should not continue disassembling after the CALL (unless2781        // there are other branches to that label).2782        if (MI->getParent()->succ_empty())2783          EmitAndCountInstruction(MCInstBuilder(X86::INT3));2784        else2785          EmitAndCountInstruction(MCInstBuilder(X86::NOOP));2786      }2787      return;2788    }2789 2790    // Set up iterator to scan the next basic block.2791    const MachineBasicBlock *NextMBB = &*MFI;2792    MBBI = NextMBB->instr_begin();2793    MBBE = NextMBB->instr_end();2794  }2795}2796 2797void X86AsmPrinter::emitLabelAndRecordForImportCallOptimization(2798    ImportCallKind Kind) {2799  assert(EnableImportCallOptimization);2800 2801  MCSymbol *CallSiteSymbol = MMI->getContext().createNamedTempSymbol("impcall");2802  OutStreamer->emitLabel(CallSiteSymbol);2803 2804  SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]2805      .push_back({CallSiteSymbol, Kind});2806}2807