brintos / llvm-project-archived public Read only

1//===- bolt/Core/BinaryContext.cpp - Low-level context --------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the BinaryContext class.10//11//===----------------------------------------------------------------------===//12 13#include "bolt/Core/BinaryContext.h"14#include "bolt/Core/BinaryEmitter.h"15#include "bolt/Core/BinaryFunction.h"16#include "bolt/Utils/CommandLineOpts.h"17#include "bolt/Utils/Utils.h"18#include "llvm/ADT/STLExtras.h"19#include "llvm/ADT/Twine.h"20#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"21#include "llvm/DebugInfo/DWARF/DWARFFormValue.h"22#include "llvm/DebugInfo/DWARF/DWARFUnit.h"23#include "llvm/MC/MCAssembler.h"24#include "llvm/MC/MCContext.h"25#include "llvm/MC/MCDisassembler/MCDisassembler.h"26#include "llvm/MC/MCInstPrinter.h"27#include "llvm/MC/MCObjectStreamer.h"28#include "llvm/MC/MCObjectWriter.h"29#include "llvm/MC/MCRegisterInfo.h"30#include "llvm/MC/MCSectionELF.h"31#include "llvm/MC/MCStreamer.h"32#include "llvm/MC/MCSubtargetInfo.h"33#include "llvm/MC/MCSymbol.h"34#include "llvm/Support/CommandLine.h"35#include "llvm/Support/Error.h"36#include "llvm/Support/FileSystem.h"37#include "llvm/Support/Regex.h"38#include <algorithm>39#include <functional>40#include <iterator>41#include <unordered_set>42 43using namespace llvm;44 45#undef  DEBUG_TYPE46#define DEBUG_TYPE "bolt"47 48namespace opts {49 50static cl::opt<bool>51    NoHugePages("no-huge-pages",52                cl::desc("use regular size pages for code alignment"),53                cl::Hidden, cl::cat(BoltCategory));54 55static cl::opt<bool>56PrintDebugInfo("print-debug-info",57  cl::desc("print debug info when printing functions"),58  cl::Hidden,59  cl::ZeroOrMore,60  cl::cat(BoltCategory));61 62cl::opt<bool> PrintRelocations(63    "print-relocations",64    cl::desc("print relocations when printing functions/objects"), cl::Hidden,65    cl::cat(BoltCategory));66 67static cl::opt<bool>68PrintMemData("print-mem-data",69  cl::desc("print memory data annotations when printing functions"),70  cl::Hidden,71  cl::ZeroOrMore,72  cl::cat(BoltCategory));73 74cl::opt<std::string> CompDirOverride(75    "comp-dir-override",76    cl::desc("overrides DW_AT_comp_dir, and provides an alternative base "77             "location, which is used with DW_AT_dwo_name to construct a path "78             "to *.dwo files."),79    cl::Hidden, cl::init(""), cl::cat(BoltCategory));80 81static cl::opt<bool> CloneConstantIsland("clone-constant-island",82                                         cl::desc("clone constant islands"),83                                         cl::Hidden, cl::init(true),84                                         cl::ZeroOrMore, cl::cat(BoltCategory));85 86static cl::opt<bool>87    FailOnInvalidPadding("fail-on-invalid-padding", cl::Hidden, cl::init(false),88                         cl::desc("treat invalid code padding as error"),89                         cl::ZeroOrMore, cl::cat(BoltCategory));90} // namespace opts91 92namespace llvm {93namespace bolt {94 95char BOLTError::ID = 0;96 97BOLTError::BOLTError(bool IsFatal, const Twine &S)98    : IsFatal(IsFatal), Msg(S.str()) {}99 100void BOLTError::log(raw_ostream &OS) const {101  if (IsFatal)102    OS << "FATAL ";103  StringRef ErrMsg = StringRef(Msg);104  // Prepend our error prefix if it is missing105  if (ErrMsg.empty()) {106    OS << "BOLT-ERROR\n";107  } else {108    if (!ErrMsg.starts_with("BOLT-ERROR"))109      OS << "BOLT-ERROR: ";110    OS << ErrMsg << "\n";111  }112}113 114std::error_code BOLTError::convertToErrorCode() const {115  return inconvertibleErrorCode();116}117 118Error createNonFatalBOLTError(const Twine &S) {119  return make_error<BOLTError>(/*IsFatal*/ false, S);120}121 122Error createFatalBOLTError(const Twine &S) {123  return make_error<BOLTError>(/*IsFatal*/ true, S);124}125 126void BinaryContext::logBOLTErrorsAndQuitOnFatal(Error E) {127  handleAllErrors(Error(std::move(E)), [&](const BOLTError &E) {128    if (!E.getMessage().empty())129      E.log(this->errs());130    if (E.isFatal())131      exit(1);132  });133}134 135BinaryContext::BinaryContext(std::unique_ptr<MCContext> Ctx,136                             std::unique_ptr<DWARFContext> DwCtx,137                             std::unique_ptr<Triple> TheTriple,138                             std::shared_ptr<orc::SymbolStringPool> SSP,139                             const Target *TheTarget, std::string TripleName,140                             std::unique_ptr<MCCodeEmitter> MCE,141                             std::unique_ptr<MCObjectFileInfo> MOFI,142                             std::unique_ptr<const MCAsmInfo> AsmInfo,143                             std::unique_ptr<const MCInstrInfo> MII,144                             std::unique_ptr<const MCSubtargetInfo> STI,145                             std::unique_ptr<MCInstPrinter> InstPrinter,146                             std::unique_ptr<const MCInstrAnalysis> MIA,147                             std::unique_ptr<MCPlusBuilder> MIB,148                             std::unique_ptr<const MCRegisterInfo> MRI,149                             std::unique_ptr<MCDisassembler> DisAsm,150                             JournalingStreams Logger)151    : Ctx(std::move(Ctx)), DwCtx(std::move(DwCtx)),152      TheTriple(std::move(TheTriple)), SSP(std::move(SSP)),153      TheTarget(TheTarget), TripleName(TripleName), MCE(std::move(MCE)),154      MOFI(std::move(MOFI)), AsmInfo(std::move(AsmInfo)), MII(std::move(MII)),155      STI(std::move(STI)), InstPrinter(std::move(InstPrinter)),156      MIA(std::move(MIA)), MIB(std::move(MIB)), MRI(std::move(MRI)),157      DisAsm(std::move(DisAsm)), Logger(Logger), InitialDynoStats(isAArch64()) {158  RegularPageSize = isAArch64() ? RegularPageSizeAArch64 : RegularPageSizeX86;159  PageAlign = opts::NoHugePages ? RegularPageSize : HugePageSize;160}161 162BinaryContext::~BinaryContext() {163  for (BinarySection *Section : Sections)164    delete Section;165  for (BinaryFunction *InjectedFunction : InjectedBinaryFunctions)166    delete InjectedFunction;167  for (std::pair<const uint64_t, JumpTable *> JTI : JumpTables)168    delete JTI.second;169  clearBinaryData();170}171 172/// Create BinaryContext for a given architecture \p ArchName and173/// triple \p TripleName.174Expected<std::unique_ptr<BinaryContext>> BinaryContext::createBinaryContext(175    Triple TheTriple, std::shared_ptr<orc::SymbolStringPool> SSP,176    StringRef InputFileName, SubtargetFeatures *Features, bool IsPIC,177    std::unique_ptr<DWARFContext> DwCtx, JournalingStreams Logger) {178  StringRef ArchName = "";179  std::string FeaturesStr = "";180  switch (TheTriple.getArch()) {181  case llvm::Triple::x86_64:182    if (Features)183      return createFatalBOLTError(184          "x86_64 target does not use SubtargetFeatures");185    ArchName = "x86-64";186    FeaturesStr = "+nopl";187    break;188  case llvm::Triple::aarch64:189    if (Features)190      return createFatalBOLTError(191          "AArch64 target does not use SubtargetFeatures");192    ArchName = "aarch64";193    FeaturesStr = "+all";194    break;195  case llvm::Triple::riscv64: {196    ArchName = "riscv64";197    if (!Features)198      return createFatalBOLTError("RISCV target needs SubtargetFeatures");199    // We rely on relaxation for some transformations (e.g., promoting all calls200    // to PseudoCALL and then making JITLink relax them). Since the relax201    // feature is not stored in the object file, we manually enable it.202    Features->AddFeature("relax");203    FeaturesStr = Features->getString();204    break;205  }206  default:207    return createStringError(std::errc::not_supported,208                             "BOLT-ERROR: Unrecognized machine in ELF file");209  }210 211  const std::string TripleName = TheTriple.str();212 213  std::string Error;214  const Target *TheTarget =215      TargetRegistry::lookupTarget(ArchName, TheTriple, Error);216  if (!TheTarget)217    return createStringError(make_error_code(std::errc::not_supported),218                             Twine("BOLT-ERROR: ", Error));219 220  std::unique_ptr<const MCRegisterInfo> MRI(221      TheTarget->createMCRegInfo(TheTriple));222  if (!MRI)223    return createStringError(224        make_error_code(std::errc::not_supported),225        Twine("BOLT-ERROR: no register info for target ", TripleName));226 227  // Set up disassembler.228  std::unique_ptr<MCAsmInfo> AsmInfo(229      TheTarget->createMCAsmInfo(*MRI, TheTriple, MCTargetOptions()));230  if (!AsmInfo)231    return createStringError(232        make_error_code(std::errc::not_supported),233        Twine("BOLT-ERROR: no assembly info for target ", TripleName));234  // BOLT creates "func@PLT" symbols for PLT entries. In function assembly dump235  // we want to emit such names as using @PLT without double quotes to convey236  // variant kind to the assembler. BOLT doesn't rely on the linker so we can237  // override the default AsmInfo behavior to emit names the way we want.238  AsmInfo->setAllowAtInName(true);239 240  std::unique_ptr<const MCSubtargetInfo> STI(241      TheTarget->createMCSubtargetInfo(TheTriple, "", FeaturesStr));242  if (!STI)243    return createStringError(244        make_error_code(std::errc::not_supported),245        Twine("BOLT-ERROR: no subtarget info for target ", TripleName));246 247  std::unique_ptr<const MCInstrInfo> MII(TheTarget->createMCInstrInfo());248  if (!MII)249    return createStringError(250        make_error_code(std::errc::not_supported),251        Twine("BOLT-ERROR: no instruction info for target ", TripleName));252 253  std::unique_ptr<MCContext> Ctx(254      new MCContext(TheTriple, AsmInfo.get(), MRI.get(), STI.get()));255  std::unique_ptr<MCObjectFileInfo> MOFI(256      TheTarget->createMCObjectFileInfo(*Ctx, IsPIC));257  Ctx->setObjectFileInfo(MOFI.get());258  // We do not support X86 Large code model. Change this in the future.259  bool Large = false;260  if (TheTriple.getArch() == llvm::Triple::aarch64)261    Large = true;262  unsigned LSDAEncoding =263      Large ? dwarf::DW_EH_PE_absptr : dwarf::DW_EH_PE_udata4;264  if (IsPIC) {265    LSDAEncoding = dwarf::DW_EH_PE_pcrel |266                   (Large ? dwarf::DW_EH_PE_sdata8 : dwarf::DW_EH_PE_sdata4);267  }268 269  std::unique_ptr<MCDisassembler> DisAsm(270      TheTarget->createMCDisassembler(*STI, *Ctx));271 272  if (!DisAsm)273    return createStringError(274        make_error_code(std::errc::not_supported),275        Twine("BOLT-ERROR: no disassembler info for target ", TripleName));276 277  std::unique_ptr<const MCInstrAnalysis> MIA(278      TheTarget->createMCInstrAnalysis(MII.get()));279  if (!MIA)280    return createStringError(281        make_error_code(std::errc::not_supported),282        Twine("BOLT-ERROR: failed to create instruction analysis for target ",283              TripleName));284 285  int AsmPrinterVariant = AsmInfo->getAssemblerDialect();286  std::unique_ptr<MCInstPrinter> InstructionPrinter(287      TheTarget->createMCInstPrinter(TheTriple, AsmPrinterVariant, *AsmInfo,288                                     *MII, *MRI));289  if (!InstructionPrinter)290    return createStringError(291        make_error_code(std::errc::not_supported),292        Twine("BOLT-ERROR: no instruction printer for target ", TripleName));293  InstructionPrinter->setPrintImmHex(true);294 295  std::unique_ptr<MCCodeEmitter> MCE(296      TheTarget->createMCCodeEmitter(*MII, *Ctx));297 298  auto BC = std::make_unique<BinaryContext>(299      std::move(Ctx), std::move(DwCtx), std::make_unique<Triple>(TheTriple),300      std::move(SSP), TheTarget, std::string(TripleName), std::move(MCE),301      std::move(MOFI), std::move(AsmInfo), std::move(MII), std::move(STI),302      std::move(InstructionPrinter), std::move(MIA), nullptr, std::move(MRI),303      std::move(DisAsm), Logger);304 305  BC->LSDAEncoding = LSDAEncoding;306 307  BC->MAB = std::unique_ptr<MCAsmBackend>(308      BC->TheTarget->createMCAsmBackend(*BC->STI, *BC->MRI, MCTargetOptions()));309 310  BC->setFilename(InputFileName);311 312  BC->HasFixedLoadAddress = !IsPIC;313 314  BC->SymbolicDisAsm = std::unique_ptr<MCDisassembler>(315      BC->TheTarget->createMCDisassembler(*BC->STI, *BC->Ctx));316 317  if (!BC->SymbolicDisAsm)318    return createStringError(319        make_error_code(std::errc::not_supported),320        Twine("BOLT-ERROR: no disassembler info for target ", TripleName));321 322  return std::move(BC);323}324 325bool BinaryContext::forceSymbolRelocations(StringRef SymbolName) const {326  if (opts::HotText &&327      (SymbolName == "__hot_start" || SymbolName == "__hot_end"))328    return true;329 330  if (opts::HotData &&331      (SymbolName == "__hot_data_start" || SymbolName == "__hot_data_end"))332    return true;333 334  if (SymbolName == "_end")335    return true;336 337  return false;338}339 340std::unique_ptr<MCObjectWriter>341BinaryContext::createObjectWriter(raw_pwrite_stream &OS) {342  return MAB->createObjectWriter(OS);343}344 345bool BinaryContext::validateObjectNesting() const {346  auto Itr = BinaryDataMap.begin();347  auto End = BinaryDataMap.end();348  bool Valid = true;349  while (Itr != End) {350    auto Next = std::next(Itr);351    while (Next != End &&352           Itr->second->getSection() == Next->second->getSection() &&353           Itr->second->containsRange(Next->second->getAddress(),354                                      Next->second->getSize())) {355      if (Next->second->Parent != Itr->second) {356        this->errs() << "BOLT-WARNING: object nesting incorrect for:\n"357                     << "BOLT-WARNING:  " << *Itr->second << "\n"358                     << "BOLT-WARNING:  " << *Next->second << "\n";359        Valid = false;360      }361      ++Next;362    }363    Itr = Next;364  }365  return Valid;366}367 368bool BinaryContext::validateHoles() const {369  bool Valid = true;370  for (BinarySection &Section : sections()) {371    for (const Relocation &Rel : Section.relocations()) {372      uint64_t RelAddr = Rel.Offset + Section.getAddress();373      const BinaryData *BD = getBinaryDataContainingAddress(RelAddr);374      if (!BD) {375        this->errs()376            << "BOLT-WARNING: no BinaryData found for relocation at address"377            << " 0x" << Twine::utohexstr(RelAddr) << " in " << Section.getName()378            << "\n";379        Valid = false;380      } else if (!BD->getAtomicRoot()) {381        this->errs()382            << "BOLT-WARNING: no atomic BinaryData found for relocation at "383            << "address 0x" << Twine::utohexstr(RelAddr) << " in "384            << Section.getName() << "\n";385        Valid = false;386      }387    }388  }389  return Valid;390}391 392void BinaryContext::updateObjectNesting(BinaryDataMapType::iterator GAI) {393  const uint64_t Address = GAI->second->getAddress();394  const uint64_t Size = GAI->second->getSize();395 396  auto fixParents = [&](BinaryDataMapType::iterator Itr,397                        BinaryData *NewParent) {398    BinaryData *OldParent = Itr->second->Parent;399    Itr->second->Parent = NewParent;400    ++Itr;401    while (Itr != BinaryDataMap.end() && OldParent &&402           Itr->second->Parent == OldParent) {403      Itr->second->Parent = NewParent;404      ++Itr;405    }406  };407 408  // Check if the previous symbol contains the newly added symbol.409  if (GAI != BinaryDataMap.begin()) {410    BinaryData *Prev = std::prev(GAI)->second;411    while (Prev) {412      if (Prev->getSection() == GAI->second->getSection() &&413          Prev->containsRange(Address, Size)) {414        fixParents(GAI, Prev);415      } else {416        fixParents(GAI, nullptr);417      }418      Prev = Prev->Parent;419    }420  }421 422  // Check if the newly added symbol contains any subsequent symbols.423  if (Size != 0) {424    BinaryData *BD = GAI->second->Parent ? GAI->second->Parent : GAI->second;425    auto Itr = std::next(GAI);426    while (427        Itr != BinaryDataMap.end() &&428        BD->containsRange(Itr->second->getAddress(), Itr->second->getSize())) {429      Itr->second->Parent = BD;430      ++Itr;431    }432  }433}434 435iterator_range<BinaryContext::binary_data_iterator>436BinaryContext::getSubBinaryData(BinaryData *BD) {437  auto Start = std::next(BinaryDataMap.find(BD->getAddress()));438  auto End = Start;439  while (End != BinaryDataMap.end() && BD->isAncestorOf(End->second))440    ++End;441  return make_range(Start, End);442}443 444std::pair<const MCSymbol *, uint64_t>445BinaryContext::handleAddressRef(uint64_t Address, BinaryFunction &BF,446                                bool IsPCRel) {447  if (isAArch64()) {448    // Check if this is an access to a constant island and create bookkeeping449    // to keep track of it and emit it later as part of this function.450    if (MCSymbol *IslandSym = BF.getOrCreateIslandAccess(Address))451      return std::make_pair(IslandSym, 0);452 453    // Detect custom code written in assembly that refers to arbitrary454    // constant islands from other functions. Write this reference so we455    // can pull this constant island and emit it as part of this function456    // too.457    auto IslandIter = AddressToConstantIslandMap.lower_bound(Address);458 459    if (IslandIter != AddressToConstantIslandMap.begin() &&460        (IslandIter == AddressToConstantIslandMap.end() ||461         IslandIter->first > Address))462      --IslandIter;463 464    if (IslandIter != AddressToConstantIslandMap.end()) {465      // Fall-back to referencing the original constant island in the presence466      // of dynamic relocs, as we currently do not support cloning them.467      // Notice: we might fail to link because of this, if the original constant468      // island we are referring would be emitted too far away.469      if (IslandIter->second->hasDynamicRelocationAtIsland() ||470          !opts::CloneConstantIsland) {471        MCSymbol *IslandSym =472            IslandIter->second->getOrCreateIslandAccess(Address);473        if (IslandSym)474          return std::make_pair(IslandSym, 0);475      } else if (MCSymbol *IslandSym =476                     IslandIter->second->getOrCreateProxyIslandAccess(Address,477                                                                      BF)) {478        LLVM_DEBUG(479            dbgs() << "BOLT-DEBUG: clone constant island at address 0x"480                   << Twine::utohexstr(IslandIter->first) << " with size of 0x"481                   << Twine::utohexstr(482                          IslandIter->second->estimateConstantIslandSize())483                   << " bytes, referenced by " << BF << "\n");484        BF.createIslandDependency(IslandSym, IslandIter->second);485        return std::make_pair(IslandSym, 0);486      }487    }488  }489 490  // Note that the address does not necessarily have to reside inside491  // a section, it could be an absolute address too.492  ErrorOr<BinarySection &> Section = getSectionForAddress(Address);493  if (Section && Section->isText()) {494    if (BF.containsAddress(Address, /*UseMaxSize=*/isAArch64())) {495      if (Address != BF.getAddress()) {496        // The address could potentially escape. Mark it as another entry497        // point into the function.498        if (opts::Verbosity >= 1) {499          this->outs() << "BOLT-INFO: potentially escaped address 0x"500                       << Twine::utohexstr(Address) << " in function " << BF501                       << '\n';502        }503        BF.HasInternalLabelReference = true;504        return std::make_pair(505            BF.addEntryPointAtOffset(Address - BF.getAddress()), 0);506      }507    } else {508      addInterproceduralReference(&BF, Address);509    }510  }511 512  // With relocations, catch jump table references outside of the basic block513  // containing the indirect jump.514  if (HasRelocations) {515    const MemoryContentsType MemType = analyzeMemoryAt(Address, BF);516    if (MemType == MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE && IsPCRel) {517      const MCSymbol *Symbol =518          getOrCreateJumpTable(BF, Address, JumpTable::JTT_PIC);519 520      return std::make_pair(Symbol, 0);521    }522  }523 524  if (BinaryData *BD = getBinaryDataContainingAddress(Address))525    return std::make_pair(BD->getSymbol(), Address - BD->getAddress());526 527  // TODO: use DWARF info to get size/alignment here?528  MCSymbol *TargetSymbol = getOrCreateGlobalSymbol(Address, "DATAat");529  LLVM_DEBUG(dbgs() << "Created symbol " << TargetSymbol->getName() << '\n');530  return std::make_pair(TargetSymbol, 0);531}532 533MCSymbol *BinaryContext::handleExternalBranchTarget(uint64_t Address,534                                                    BinaryFunction &BF) {535  if (BF.isInConstantIsland(Address)) {536    BF.setIgnored();537    this->outs() << "BOLT-WARNING: ignoring entry point at address 0x"538                 << Twine::utohexstr(Address)539                 << " in constant island of function " << BF << '\n';540    return nullptr;541  }542 543  const uint64_t Offset = Address - BF.getAddress();544  assert(Offset < BF.getSize() &&545         "Address should be inside the referenced function");546 547  return Offset ? BF.addEntryPointAtOffset(Offset) : BF.getSymbol();548}549 550MemoryContentsType BinaryContext::analyzeMemoryAt(uint64_t Address,551                                                  BinaryFunction &BF) {552  if (!isX86())553    return MemoryContentsType::UNKNOWN;554 555  ErrorOr<BinarySection &> Section = getSectionForAddress(Address);556  if (!Section) {557    // No section - possibly an absolute address. Since we don't allow558    // internal function addresses to escape the function scope - we559    // consider it a tail call.560    if (opts::Verbosity > 1) {561      this->errs() << "BOLT-WARNING: no section for address 0x"562                   << Twine::utohexstr(Address) << " referenced from function "563                   << BF << '\n';564    }565    return MemoryContentsType::UNKNOWN;566  }567 568  if (Section->isVirtual()) {569    // The contents are filled at runtime.570    return MemoryContentsType::UNKNOWN;571  }572 573  // No support for jump tables in code yet.574  if (Section->isText())575    return MemoryContentsType::UNKNOWN;576 577  // Start with checking for PIC jump table. We expect non-PIC jump tables578  // to have high 32 bits set to 0.579  if (analyzeJumpTable(Address, JumpTable::JTT_PIC, BF))580    return MemoryContentsType::POSSIBLE_PIC_JUMP_TABLE;581 582  if (analyzeJumpTable(Address, JumpTable::JTT_NORMAL, BF))583    return MemoryContentsType::POSSIBLE_JUMP_TABLE;584 585  return MemoryContentsType::UNKNOWN;586}587 588bool BinaryContext::analyzeJumpTable(const uint64_t Address,589                                     const JumpTable::JumpTableType Type,590                                     const BinaryFunction &BF,591                                     const uint64_t NextJTAddress,592                                     JumpTable::AddressesType *EntriesAsAddress,593                                     bool *HasEntryInFragment) const {594  // Target address of __builtin_unreachable.595  const uint64_t UnreachableAddress = BF.getAddress() + BF.getSize();596 597  // Is one of the targets __builtin_unreachable?598  bool HasUnreachable = false;599 600  // Does one of the entries match function start address?601  bool HasStartAsEntry = false;602 603  // Number of targets other than __builtin_unreachable.604  uint64_t NumRealEntries = 0;605 606  // Size of the jump table without trailing __builtin_unreachable entries.607  size_t TrimmedSize = 0;608 609  auto addEntryAddress = [&](uint64_t EntryAddress, bool Unreachable = false) {610    if (!EntriesAsAddress)611      return;612    EntriesAsAddress->emplace_back(EntryAddress);613    if (!Unreachable)614      TrimmedSize = EntriesAsAddress->size();615  };616 617  auto printEntryDiagnostics = [&](raw_ostream &OS,618                                   const BinaryFunction *TargetBF) {619    OS << "FAIL: function doesn't contain this address\n";620    if (!TargetBF)621      return;622    OS << "  ! function containing this address: " << *TargetBF << '\n';623    if (!TargetBF->isFragment())624      return;625    OS << "  ! is a fragment with parents: ";626    ListSeparator LS;627    for (BinaryFunction *Parent : TargetBF->ParentFragments)628      OS << LS << *Parent;629    OS << '\n';630  };631 632  ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);633  if (!Section)634    return false;635 636  // The upper bound is defined by containing object, section limits, and637  // the next jump table in memory.638  uint64_t UpperBound = Section->getEndAddress();639  const BinaryData *JumpTableBD = getBinaryDataAtAddress(Address);640  if (JumpTableBD && JumpTableBD->getSize()) {641    assert(JumpTableBD->getEndAddress() <= UpperBound &&642           "data object cannot cross a section boundary");643    UpperBound = JumpTableBD->getEndAddress();644  }645  if (NextJTAddress)646    UpperBound = std::min(NextJTAddress, UpperBound);647 648  LLVM_DEBUG({649    using JTT = JumpTable::JumpTableType;650    dbgs() << formatv("BOLT-DEBUG: analyzeJumpTable @{0:x} in {1}, JTT={2}\n",651                      Address, BF.getPrintName(),652                      Type == JTT::JTT_PIC ? "PIC" : "Normal");653  });654  const uint64_t EntrySize = getJumpTableEntrySize(Type);655  for (uint64_t EntryAddress = Address; EntryAddress <= UpperBound - EntrySize;656       EntryAddress += EntrySize) {657    LLVM_DEBUG(dbgs() << "  * Checking 0x" << Twine::utohexstr(EntryAddress)658                      << " -> ");659    // Check if there's a proper relocation against the jump table entry.660    if (HasRelocations) {661      if (Type == JumpTable::JTT_PIC &&662          !DataPCRelocations.count(EntryAddress)) {663        LLVM_DEBUG(664            dbgs() << "FAIL: JTT_PIC table, no relocation for this address\n");665        break;666      }667      if (Type == JumpTable::JTT_NORMAL && !getRelocationAt(EntryAddress)) {668        LLVM_DEBUG(669            dbgs()670            << "FAIL: JTT_NORMAL table, no relocation for this address\n");671        break;672      }673    }674 675    const uint64_t Value =676        (Type == JumpTable::JTT_PIC)677            ? Address + *getSignedValueAtAddress(EntryAddress, EntrySize)678            : *getPointerAtAddress(EntryAddress);679 680    // __builtin_unreachable() case.681    if (Value == UnreachableAddress) {682      addEntryAddress(Value, /*Unreachable*/ true);683      HasUnreachable = true;684      LLVM_DEBUG(dbgs() << formatv("OK: {0:x} __builtin_unreachable\n", Value));685      continue;686    }687 688    // Function start is another special case. It is allowed in the jump table,689    // but we need at least one another regular entry to distinguish the table690    // from, e.g. a function pointer array.691    if (Value == BF.getAddress()) {692      HasStartAsEntry = true;693      addEntryAddress(Value);694      continue;695    }696 697    // Function or one of its fragments.698    const BinaryFunction *TargetBF = getBinaryFunctionContainingAddress(Value);699    if (!TargetBF || !areRelatedFragments(TargetBF, &BF)) {700      LLVM_DEBUG(printEntryDiagnostics(dbgs(), TargetBF));701      (void)printEntryDiagnostics;702      break;703    }704 705    // Check there's an instruction at this offset.706    if (TargetBF->getState() == BinaryFunction::State::Disassembled &&707        !TargetBF->getInstructionAtOffset(Value - TargetBF->getAddress())) {708      LLVM_DEBUG(dbgs() << formatv("FAIL: no instruction at {0:x}\n", Value));709      break;710    }711 712    ++NumRealEntries;713    LLVM_DEBUG(dbgs() << formatv("OK: {0:x} real entry\n", Value));714 715    if (TargetBF != &BF && HasEntryInFragment)716      *HasEntryInFragment = true;717    addEntryAddress(Value);718  }719 720  // Trim direct/normal jump table to exclude trailing unreachable entries that721  // can collide with a function address.722  if (Type == JumpTable::JTT_NORMAL && EntriesAsAddress &&723      TrimmedSize != EntriesAsAddress->size() &&724      getBinaryFunctionAtAddress(UnreachableAddress))725    EntriesAsAddress->resize(TrimmedSize);726 727  // It's a jump table if the number of real entries is more than 1, or there's728  // one real entry and one or more special targets. If there are only multiple729  // special targets, then it's not a jump table.730  return NumRealEntries + (HasUnreachable || HasStartAsEntry) >= 2;731}732 733void BinaryContext::populateJumpTables() {734  LLVM_DEBUG(dbgs() << "DataPCRelocations: " << DataPCRelocations.size()735                    << '\n');736  for (auto JTI = JumpTables.begin(), JTE = JumpTables.end(); JTI != JTE;737       ++JTI) {738    JumpTable *JT = JTI->second;739 740    if (!llvm::all_of(JT->Parents, std::mem_fn(&BinaryFunction::isSimple)))741      continue;742 743    uint64_t NextJTAddress = 0;744    auto NextJTI = std::next(JTI);745    if (NextJTI != JTE)746      NextJTAddress = NextJTI->second->getAddress();747 748    const bool Success =749        analyzeJumpTable(JT->getAddress(), JT->Type, *(JT->Parents[0]),750                         NextJTAddress, &JT->EntriesAsAddress, &JT->IsSplit);751    if (!Success) {752      LLVM_DEBUG({753        dbgs() << "failed to analyze ";754        JT->print(dbgs());755        if (NextJTI != JTE) {756          dbgs() << "next ";757          NextJTI->second->print(dbgs());758        }759      });760      llvm_unreachable("jump table heuristic failure");761    }762    for (BinaryFunction *Frag : JT->Parents) {763      if (JT->IsSplit)764        Frag->setHasIndirectTargetToSplitFragment(true);765      for (uint64_t EntryAddress : JT->EntriesAsAddress)766        // if target is builtin_unreachable767        if (EntryAddress == Frag->getAddress() + Frag->getSize()) {768          Frag->IgnoredBranches.emplace_back(EntryAddress - Frag->getAddress(),769                                             Frag->getSize());770        } else if (EntryAddress >= Frag->getAddress() &&771                   EntryAddress < Frag->getAddress() + Frag->getSize()) {772          Frag->registerReferencedOffset(EntryAddress - Frag->getAddress());773        }774    }775 776    // In strict mode, erase PC-relative relocation record. Later we check that777    // all such records are erased and thus have been accounted for.778    if (opts::StrictMode && JT->Type == JumpTable::JTT_PIC) {779      for (uint64_t Address = JT->getAddress();780           Address < JT->getAddress() + JT->getSize();781           Address += JT->EntrySize) {782        DataPCRelocations.erase(DataPCRelocations.find(Address));783      }784    }785 786    // Mark to skip the function and all its fragments.787    for (BinaryFunction *Frag : JT->Parents)788      if (Frag->hasIndirectTargetToSplitFragment())789        addFragmentsToSkip(Frag);790  }791 792  if (opts::StrictMode && DataPCRelocations.size()) {793    this->errs() << "BOLT-ERROR: " << DataPCRelocations.size()794                 << " unclaimed PC-relative relocation(s) left in data";795    if (opts::Verbosity) {796      this->errs() << ":\n";797      for (uint64_t RelocOffset : DataPCRelocations)798        this->errs() << "  @0x" << Twine::utohexstr(RelocOffset) << '\n';799    } else {800      this->errs() << ". Re-run with -v=1 to see the list\n";801    }802    this->errs() << "BOLT-ERROR: unable to proceed with --strict\n";803    exit(1);804  }805  clearList(DataPCRelocations);806}807 808void BinaryContext::skipMarkedFragments() {809  std::vector<BinaryFunction *> FragmentQueue;810  // Copy the functions to FragmentQueue.811  FragmentQueue.assign(FragmentsToSkip.begin(), FragmentsToSkip.end());812  auto addToWorklist = [&](BinaryFunction *Function) -> void {813    if (FragmentsToSkip.count(Function))814      return;815    FragmentQueue.push_back(Function);816    addFragmentsToSkip(Function);817  };818  // Functions containing split jump tables need to be skipped with all819  // fragments (transitively).820  for (size_t I = 0; I != FragmentQueue.size(); I++) {821    BinaryFunction *BF = FragmentQueue[I];822    assert(FragmentsToSkip.count(BF) &&823           "internal error in traversing function fragments");824    if (opts::Verbosity >= 1)825      this->errs() << "BOLT-WARNING: Ignoring " << BF->getPrintName() << '\n';826    BF->setSimple(false);827    BF->setHasIndirectTargetToSplitFragment(true);828 829    llvm::for_each(BF->Fragments, addToWorklist);830    llvm::for_each(BF->ParentFragments, addToWorklist);831  }832  if (!FragmentsToSkip.empty())833    this->errs() << "BOLT-WARNING: skipped " << FragmentsToSkip.size()834                 << " function" << (FragmentsToSkip.size() == 1 ? "" : "s")835                 << " due to cold fragments\n";836}837 838MCSymbol *BinaryContext::getOrCreateGlobalSymbol(uint64_t Address, Twine Prefix,839                                                 uint64_t Size,840                                                 uint16_t Alignment,841                                                 unsigned Flags) {842  auto Itr = BinaryDataMap.find(Address);843  if (Itr != BinaryDataMap.end()) {844    assert(Itr->second->getSize() == Size || !Size);845    return Itr->second->getSymbol();846  }847 848  std::string Name = (Prefix + "0x" + Twine::utohexstr(Address)).str();849  assert(!GlobalSymbols.count(Name) && "created name is not unique");850  return registerNameAtAddress(Name, Address, Size, Alignment, Flags);851}852 853MCSymbol *BinaryContext::getOrCreateUndefinedGlobalSymbol(StringRef Name) {854  return Ctx->getOrCreateSymbol(Name);855}856 857BinaryFunction *BinaryContext::createBinaryFunction(858    const std::string &Name, BinarySection &Section, uint64_t Address,859    uint64_t Size, uint64_t SymbolSize, uint16_t Alignment) {860  auto Result = BinaryFunctions.emplace(861      Address, BinaryFunction(Name, Section, Address, Size, *this));862  assert(Result.second == true && "unexpected duplicate function");863  BinaryFunction *BF = &Result.first->second;864  registerNameAtAddress(Name, Address, SymbolSize ? SymbolSize : Size,865                        Alignment);866  setSymbolToFunctionMap(BF->getSymbol(), BF);867  return BF;868}869 870const MCSymbol *871BinaryContext::getOrCreateJumpTable(BinaryFunction &Function, uint64_t Address,872                                    JumpTable::JumpTableType Type) {873  // Two fragments of same function access same jump table874  if (JumpTable *JT = getJumpTableContainingAddress(Address)) {875    assert(JT->Type == Type && "jump table types have to match");876    assert(Address == JT->getAddress() && "unexpected non-empty jump table");877 878    if (llvm::is_contained(JT->Parents, &Function))879      return JT->getFirstLabel();880 881    // Prevent associating a jump table to a specific fragment twice.882    auto isSibling = std::bind(&BinaryContext::areRelatedFragments, this,883                               &Function, std::placeholders::_1);884    assert(llvm::all_of(JT->Parents, isSibling) &&885           "cannot reuse jump table of a different function");886    (void)isSibling;887    if (opts::Verbosity > 2) {888      this->outs() << "BOLT-INFO: multiple fragments access the same jump table"889                   << ": " << *JT->Parents[0] << "; " << Function << '\n';890      JT->print(this->outs());891    }892    if (JT->Parents.size() == 1)893      JT->Parents.front()->setHasIndirectTargetToSplitFragment(true);894    Function.setHasIndirectTargetToSplitFragment(true);895    // Duplicate the entry for the parent function for easy access896    JT->Parents.push_back(&Function);897    Function.JumpTables.emplace(Address, JT);898    return JT->getFirstLabel();899  }900 901  // Reuse the existing symbol if possible.902  MCSymbol *JTLabel = nullptr;903  if (BinaryData *Object = getBinaryDataAtAddress(Address)) {904    if (!isInternalSymbolName(Object->getSymbol()->getName()))905      JTLabel = Object->getSymbol();906  }907 908  const uint64_t EntrySize = getJumpTableEntrySize(Type);909  if (!JTLabel) {910    const std::string JumpTableName = generateJumpTableName(Function, Address);911    JTLabel = registerNameAtAddress(JumpTableName, Address, 0, EntrySize);912  }913 914  LLVM_DEBUG(dbgs() << "BOLT-DEBUG: creating jump table " << JTLabel->getName()915                    << " in function " << Function << '\n');916 917  JumpTable *JT = new JumpTable(*JTLabel, Address, EntrySize, Type,918                                JumpTable::LabelMapType{{0, JTLabel}},919                                *getSectionForAddress(Address));920  JT->Parents.push_back(&Function);921  if (opts::Verbosity > 2)922    JT->print(this->outs());923  JumpTables.emplace(Address, JT);924 925  // Duplicate the entry for the parent function for easy access.926  Function.JumpTables.emplace(Address, JT);927  return JTLabel;928}929 930std::pair<uint64_t, const MCSymbol *>931BinaryContext::duplicateJumpTable(BinaryFunction &Function, JumpTable *JT,932                                  const MCSymbol *OldLabel) {933  auto L = scopeLock();934  unsigned Offset = 0;935  bool Found = false;936  for (std::pair<const unsigned, MCSymbol *> Elmt : JT->Labels) {937    if (Elmt.second != OldLabel)938      continue;939    Offset = Elmt.first;940    Found = true;941    break;942  }943  assert(Found && "Label not found");944  (void)Found;945  MCSymbol *NewLabel = Ctx->createNamedTempSymbol("duplicatedJT");946  JumpTable *NewJT =947      new JumpTable(*NewLabel, JT->getAddress(), JT->EntrySize, JT->Type,948                    JumpTable::LabelMapType{{Offset, NewLabel}},949                    *getSectionForAddress(JT->getAddress()));950  NewJT->Parents = JT->Parents;951  NewJT->Entries = JT->Entries;952  NewJT->Counts = JT->Counts;953  uint64_t JumpTableID = ++DuplicatedJumpTables;954  // Invert it to differentiate from regular jump tables whose IDs are their955  // addresses in the input binary memory space956  JumpTableID = ~JumpTableID;957  JumpTables.emplace(JumpTableID, NewJT);958  Function.JumpTables.emplace(JumpTableID, NewJT);959  return std::make_pair(JumpTableID, NewLabel);960}961 962std::string BinaryContext::generateJumpTableName(const BinaryFunction &BF,963                                                 uint64_t Address) {964  size_t Id;965  uint64_t Offset = 0;966  if (const JumpTable *JT = BF.getJumpTableContainingAddress(Address)) {967    Offset = Address - JT->getAddress();968    auto JTLabelsIt = JT->Labels.find(Offset);969    if (JTLabelsIt != JT->Labels.end())970      return std::string(JTLabelsIt->second->getName());971 972    auto JTIdsIt = JumpTableIds.find(JT->getAddress());973    assert(JTIdsIt != JumpTableIds.end());974    Id = JTIdsIt->second;975  } else {976    Id = JumpTableIds[Address] = BF.JumpTables.size();977  }978  return ("JUMP_TABLE/" + BF.getOneName().str() + "." + std::to_string(Id) +979          (Offset ? ("." + std::to_string(Offset)) : ""));980}981 982bool BinaryContext::hasValidCodePadding(const BinaryFunction &BF) {983  if (!isX86() && !isAArch64())984    return true;985 986  if (BF.getSize() == BF.getMaxSize())987    return true;988 989  ErrorOr<ArrayRef<unsigned char>> FunctionData = BF.getData();990  assert(FunctionData && "cannot get function as data");991 992  uint64_t Offset = BF.getSize();993  MCInst Instr;994  uint64_t InstrSize = 0;995  uint64_t InstrAddress = BF.getAddress() + Offset;996  using std::placeholders::_1;997 998  // Skip instructions that satisfy the predicate condition.999  auto skipInstructions = [&](std::function<bool(const MCInst &)> Predicate) {1000    const uint64_t StartOffset = Offset;1001    for (; Offset < BF.getMaxSize();1002         Offset += InstrSize, InstrAddress += InstrSize) {1003      if (!DisAsm->getInstruction(Instr, InstrSize, FunctionData->slice(Offset),1004                                  InstrAddress, nulls()))1005        break;1006      if (!Predicate(Instr))1007        break;1008    }1009 1010    return Offset - StartOffset;1011  };1012 1013  // Skip a sequence of zero bytes. For AArch64 we only skip 4's exact1014  // multiple number of zeros in case the following zeros belong to veneer.1015  auto skipZeros = [&]() {1016    const uint64_t StartOffset = Offset;1017    uint64_t CurrentOffset = Offset;1018    for (; CurrentOffset < BF.getMaxSize(); ++CurrentOffset)1019      if ((*FunctionData)[CurrentOffset] != 0)1020        break;1021 1022    uint64_t NumZeros = CurrentOffset - StartOffset;1023    if (isAArch64())1024      NumZeros &= ~((uint64_t)0x3);1025 1026    if (NumZeros == 0)1027      return false;1028    Offset += NumZeros;1029    InstrAddress += NumZeros;1030    return true;1031  };1032 1033  // Accept the whole padding area filled with breakpoints.1034  auto isBreakpoint = std::bind(&MCPlusBuilder::isBreakpoint, MIB.get(), _1);1035  if (skipInstructions(isBreakpoint) && Offset == BF.getMaxSize())1036    return true;1037 1038  auto isNoop = std::bind(&MCPlusBuilder::isNoop, MIB.get(), _1);1039 1040  // Some functions have a jump to the next function or to the padding area1041  // inserted after the body.1042  auto isSkipJump = [&](const MCInst &Instr) {1043    if (!isX86())1044      return false;1045    uint64_t TargetAddress = 0;1046    if (MIB->isUnconditionalBranch(Instr) &&1047        MIB->evaluateBranch(Instr, InstrAddress, InstrSize, TargetAddress)) {1048      if (TargetAddress >= InstrAddress + InstrSize &&1049          TargetAddress <= BF.getAddress() + BF.getMaxSize()) {1050        return true;1051      }1052    }1053    return false;1054  };1055 1056  // For veneers that are not already covered by binary functions, only those1057  // that handleAArch64Veneer() can recognize are checked here.1058  auto skipAArch64Veneer = [&]() {1059    if (!isAArch64() || Offset >= BF.getMaxSize())1060      return false;1061    BinaryFunction *BFVeneer = getBinaryFunctionContainingAddress(InstrAddress);1062    if (BFVeneer) {1063      // A binary function may have been created to point to this veneer.1064      Offset += BFVeneer->getSize();1065      assert(Offset <= BF.getMaxSize() &&1066             "AArch64 veneeer goes past the max size of function");1067      InstrAddress += BFVeneer->getSize();1068      return true;1069    }1070    const uint64_t AArch64VeneerSize = 12;1071    if (Offset + AArch64VeneerSize <= BF.getMaxSize() &&1072        handleAArch64Veneer(InstrAddress, /*MatchOnly*/ true)) {1073      Offset += AArch64VeneerSize;1074      InstrAddress += AArch64VeneerSize;1075      this->errs() << "BOLT-WARNING: found unmarked AArch64 veneer at 0x"1076                   << Twine::utohexstr(BF.getAddress() + Offset) << '\n';1077      return true;1078    }1079    return false;1080  };1081 1082  auto skipAArch64ConstantIsland = [&]() {1083    if (!isAArch64() || Offset >= BF.getMaxSize())1084      return false;1085    uint64_t Size;1086    if (BF.isInConstantIsland(InstrAddress, &Size)) {1087      Offset += Size;1088      InstrAddress += Size;1089      return true;1090    }1091    return false;1092  };1093 1094  // Skip over nops, jumps, and zero padding. Allow interleaving (this happens).1095  // For AArch64 also check veneers and skip constant islands.1096  while (skipAArch64Veneer() || skipAArch64ConstantIsland() ||1097         skipInstructions(isNoop) || skipInstructions(isSkipJump) ||1098         skipZeros())1099    ;1100 1101  if (Offset == BF.getMaxSize())1102    return true;1103 1104  this->errs() << "BOLT-WARNING: bad padding at address 0x"1105               << Twine::utohexstr(BF.getAddress() + BF.getSize())1106               << " starting at offset " << (Offset - BF.getSize())1107               << " in function " << BF << '\n'1108               << FunctionData->slice(BF.getSize(),1109                                      BF.getMaxSize() - BF.getSize())1110               << '\n';1111  return false;1112}1113 1114void BinaryContext::adjustCodePadding() {1115  uint64_t NumInvalid = 0;1116  for (auto &BFI : BinaryFunctions) {1117    BinaryFunction &BF = BFI.second;1118    if (!shouldEmit(BF))1119      continue;1120 1121    if (!hasValidCodePadding(BF)) {1122      NumInvalid++;1123      if (HasRelocations) {1124        this->errs() << "BOLT-WARNING: function " << BF1125                     << " has invalid padding. Ignoring the function\n";1126        BF.setIgnored();1127      } else {1128        BF.setMaxSize(BF.getSize());1129      }1130    }1131  }1132  if (NumInvalid && opts::FailOnInvalidPadding) {1133    this->errs() << "BOLT-ERROR: found " << NumInvalid1134                 << " instance(s) of invalid code padding\n";1135    exit(1);1136  }1137}1138 1139MCSymbol *BinaryContext::registerNameAtAddress(StringRef Name, uint64_t Address,1140                                               uint64_t Size,1141                                               uint16_t Alignment,1142                                               unsigned Flags) {1143  // Register the name with MCContext.1144  MCSymbol *Symbol = Ctx->getOrCreateSymbol(Name);1145 1146  auto GAI = BinaryDataMap.find(Address);1147  BinaryData *BD;1148  if (GAI == BinaryDataMap.end()) {1149    ErrorOr<BinarySection &> SectionOrErr = getSectionForAddress(Address);1150    BinarySection &Section =1151        SectionOrErr ? SectionOrErr.get() : absoluteSection();1152    BD = new BinaryData(*Symbol, Address, Size, Alignment ? Alignment : 1,1153                        Section, Flags);1154    GAI = BinaryDataMap.emplace(Address, BD).first;1155    GlobalSymbols[Name] = BD;1156    updateObjectNesting(GAI);1157  } else {1158    BD = GAI->second;1159    if (!BD->hasName(Name)) {1160      GlobalSymbols[Name] = BD;1161      BD->updateSize(Size);1162      BD->Symbols.push_back(Symbol);1163    }1164  }1165 1166  return Symbol;1167}1168 1169const BinaryData *1170BinaryContext::getBinaryDataContainingAddressImpl(uint64_t Address) const {1171  auto NI = BinaryDataMap.lower_bound(Address);1172  auto End = BinaryDataMap.end();1173  if ((NI != End && Address == NI->first) ||1174      ((NI != BinaryDataMap.begin()) && (NI-- != BinaryDataMap.begin()))) {1175    if (NI->second->containsAddress(Address))1176      return NI->second;1177 1178    // If this is a sub-symbol, see if a parent data contains the address.1179    const BinaryData *BD = NI->second->getParent();1180    while (BD) {1181      if (BD->containsAddress(Address))1182        return BD;1183      BD = BD->getParent();1184    }1185  }1186  return nullptr;1187}1188 1189BinaryData *BinaryContext::getGOTSymbol() {1190  // First tries to find a global symbol with that name1191  BinaryData *GOTSymBD = getBinaryDataByName("_GLOBAL_OFFSET_TABLE_");1192  if (GOTSymBD)1193    return GOTSymBD;1194 1195  // This symbol might be hidden from run-time link, so fetch the local1196  // definition if available.1197  GOTSymBD = getBinaryDataByName("_GLOBAL_OFFSET_TABLE_/1");1198  if (!GOTSymBD)1199    return nullptr;1200 1201  // If the local symbol is not unique, fail1202  unsigned Index = 2;1203  SmallString<30> Storage;1204  while (const BinaryData *BD =1205             getBinaryDataByName(Twine("_GLOBAL_OFFSET_TABLE_/")1206                                     .concat(Twine(Index++))1207                                     .toStringRef(Storage)))1208    if (BD->getAddress() != GOTSymBD->getAddress())1209      return nullptr;1210 1211  return GOTSymBD;1212}1213 1214bool BinaryContext::setBinaryDataSize(uint64_t Address, uint64_t Size) {1215  auto NI = BinaryDataMap.find(Address);1216  assert(NI != BinaryDataMap.end());1217  if (NI == BinaryDataMap.end())1218    return false;1219  // TODO: it's possible that a jump table starts at the same address1220  // as a larger blob of private data.  When we set the size of the1221  // jump table, it might be smaller than the total blob size.  In this1222  // case we just leave the original size since (currently) it won't really1223  // affect anything.1224  assert((!NI->second->Size || NI->second->Size == Size ||1225          (NI->second->isJumpTable() && NI->second->Size > Size)) &&1226         "can't change the size of a symbol that has already had its "1227         "size set");1228  if (!NI->second->Size) {1229    NI->second->Size = Size;1230    updateObjectNesting(NI);1231    return true;1232  }1233  return false;1234}1235 1236void BinaryContext::generateSymbolHashes() {1237  auto isPadding = [](const BinaryData &BD) {1238    StringRef Contents = BD.getSection().getContents();1239    StringRef SymData = Contents.substr(BD.getOffset(), BD.getSize());1240    return (BD.getName().starts_with("HOLEat") ||1241            SymData.find_first_not_of(0) == StringRef::npos);1242  };1243 1244  uint64_t NumCollisions = 0;1245  for (auto &Entry : BinaryDataMap) {1246    BinaryData &BD = *Entry.second;1247    StringRef Name = BD.getName();1248 1249    if (!isInternalSymbolName(Name))1250      continue;1251 1252    // First check if a non-anonymous alias exists and move it to the front.1253    if (BD.getSymbols().size() > 1) {1254      auto Itr = llvm::find_if(BD.getSymbols(), [&](const MCSymbol *Symbol) {1255        return !isInternalSymbolName(Symbol->getName());1256      });1257      if (Itr != BD.getSymbols().end()) {1258        size_t Idx = std::distance(BD.getSymbols().begin(), Itr);1259        std::swap(BD.getSymbols()[0], BD.getSymbols()[Idx]);1260        continue;1261      }1262    }1263 1264    // We have to skip 0 size symbols since they will all collide.1265    if (BD.getSize() == 0) {1266      continue;1267    }1268 1269    const uint64_t Hash = BD.getSection().hash(BD);1270    const size_t Idx = Name.find("0x");1271    std::string NewName =1272        (Twine(Name.substr(0, Idx)) + "_" + Twine::utohexstr(Hash)).str();1273    if (getBinaryDataByName(NewName)) {1274      // Ignore collisions for symbols that appear to be padding1275      // (i.e. all zeros or a "hole")1276      if (!isPadding(BD)) {1277        if (opts::Verbosity) {1278          this->errs() << "BOLT-WARNING: collision detected when hashing " << BD1279                       << " with new name (" << NewName << "), skipping.\n";1280        }1281        ++NumCollisions;1282      }1283      continue;1284    }1285    BD.Symbols.insert(BD.Symbols.begin(), Ctx->getOrCreateSymbol(NewName));1286    GlobalSymbols[NewName] = &BD;1287  }1288  if (NumCollisions) {1289    this->errs() << "BOLT-WARNING: " << NumCollisions1290                 << " collisions detected while hashing binary objects";1291    if (!opts::Verbosity)1292      this->errs() << ". Use -v=1 to see the list.";1293    this->errs() << '\n';1294  }1295}1296 1297bool BinaryContext::registerFragment(BinaryFunction &TargetFunction,1298                                     BinaryFunction &Function) {1299  assert(TargetFunction.isFragment() && "TargetFunction must be a fragment");1300  if (TargetFunction.isChildOf(Function))1301    return true;1302  TargetFunction.addParentFragment(Function);1303  Function.addFragment(TargetFunction);1304  FragmentClasses.unionSets(&TargetFunction, &Function);1305  if (!HasRelocations) {1306    TargetFunction.setSimple(false);1307    Function.setSimple(false);1308  }1309  if (opts::Verbosity >= 1) {1310    this->outs() << "BOLT-INFO: marking " << TargetFunction1311                 << " as a fragment of " << Function << '\n';1312  }1313  return true;1314}1315 1316void BinaryContext::addAdrpAddRelocAArch64(BinaryFunction &BF,1317                                           MCInst &LoadLowBits,1318                                           MCInst &LoadHiBits,1319                                           uint64_t Target) {1320  const MCSymbol *TargetSymbol;1321  uint64_t Addend = 0;1322  std::tie(TargetSymbol, Addend) = handleAddressRef(Target, BF,1323                                                    /*IsPCRel*/ true);1324  int64_t Val;1325  MIB->replaceImmWithSymbolRef(LoadHiBits, TargetSymbol, Addend, Ctx.get(), Val,1326                               ELF::R_AARCH64_ADR_PREL_PG_HI21);1327  MIB->replaceImmWithSymbolRef(LoadLowBits, TargetSymbol, Addend, Ctx.get(),1328                               Val, ELF::R_AARCH64_ADD_ABS_LO12_NC);1329}1330 1331bool BinaryContext::handleAArch64Veneer(uint64_t Address, bool MatchOnly) {1332  BinaryFunction *TargetFunction = getBinaryFunctionContainingAddress(Address);1333  if (TargetFunction)1334    return false;1335 1336  ErrorOr<BinarySection &> Section = getSectionForAddress(Address);1337  assert(Section && "cannot get section for referenced address");1338  if (!Section->isText())1339    return false;1340 1341  bool Ret = false;1342  StringRef SectionContents = Section->getContents();1343  uint64_t Offset = Address - Section->getAddress();1344  const uint64_t MaxSize = SectionContents.size() - Offset;1345  const uint8_t *Bytes =1346      reinterpret_cast<const uint8_t *>(SectionContents.data());1347  ArrayRef<uint8_t> Data(Bytes + Offset, MaxSize);1348 1349  auto matchVeneer = [&](BinaryFunction::InstrMapType &Instructions,1350                         MCInst &Instruction, uint64_t Offset,1351                         uint64_t AbsoluteInstrAddr,1352                         uint64_t TotalSize) -> bool {1353    MCInst *TargetHiBits, *TargetLowBits;1354    uint64_t TargetAddress, Count;1355    Count = MIB->matchLinkerVeneer(Instructions.begin(), Instructions.end(),1356                                   AbsoluteInstrAddr, Instruction, TargetHiBits,1357                                   TargetLowBits, TargetAddress);1358    if (!Count)1359      return false;1360 1361    if (MatchOnly)1362      return true;1363 1364    // NOTE The target symbol was created during disassemble's1365    // handleExternalReference1366    const MCSymbol *VeneerSymbol = getOrCreateGlobalSymbol(Address, "FUNCat");1367    BinaryFunction *Veneer = createBinaryFunction(VeneerSymbol->getName().str(),1368                                                  *Section, Address, TotalSize);1369    addAdrpAddRelocAArch64(*Veneer, *TargetLowBits, *TargetHiBits,1370                           TargetAddress);1371    MIB->addAnnotation(Instruction, "AArch64Veneer", true);1372    Veneer->addInstruction(Offset, std::move(Instruction));1373    --Count;1374    for (auto It = Instructions.rbegin(); Count != 0; ++It, --Count) {1375      MIB->addAnnotation(It->second, "AArch64Veneer", true);1376      Veneer->addInstruction(It->first, std::move(It->second));1377    }1378 1379    Veneer->getOrCreateLocalLabel(Address);1380    Veneer->setMaxSize(TotalSize);1381    Veneer->updateState(BinaryFunction::State::Disassembled);1382    LLVM_DEBUG(dbgs() << "BOLT-DEBUG: handling veneer function at 0x"1383                      << Twine::utohexstr(Address) << "\n");1384    return true;1385  };1386 1387  uint64_t Size = 0, TotalSize = 0;1388  BinaryFunction::InstrMapType VeneerInstructions;1389  for (Offset = 0; Offset < MaxSize; Offset += Size) {1390    MCInst Instruction;1391    const uint64_t AbsoluteInstrAddr = Address + Offset;1392    if (!SymbolicDisAsm->getInstruction(Instruction, Size, Data.slice(Offset),1393                                        AbsoluteInstrAddr, nulls()))1394      break;1395 1396    TotalSize += Size;1397    if (MIB->isBranch(Instruction)) {1398      Ret = matchVeneer(VeneerInstructions, Instruction, Offset,1399                        AbsoluteInstrAddr, TotalSize);1400      break;1401    }1402 1403    VeneerInstructions.emplace(Offset, std::move(Instruction));1404  }1405 1406  return Ret;1407}1408 1409void BinaryContext::processInterproceduralReferences() {1410  for (const std::pair<BinaryFunction *, uint64_t> &It :1411       InterproceduralReferences) {1412    BinaryFunction &Function = *It.first;1413    uint64_t Address = It.second;1414    // Process interprocedural references from ignored functions in BAT mode1415    // (non-simple in non-relocation mode) to properly register entry points1416    if (!Address || (Function.isIgnored() && !HasBATSection))1417      continue;1418 1419    BinaryFunction *TargetFunction =1420        getBinaryFunctionContainingAddress(Address);1421    if (&Function == TargetFunction)1422      continue;1423 1424    if (TargetFunction) {1425      if (TargetFunction->isFragment() &&1426          !areRelatedFragments(TargetFunction, &Function)) {1427        this->errs()1428            << "BOLT-WARNING: interprocedural reference between unrelated "1429               "fragments: "1430            << Function.getPrintName() << " and "1431            << TargetFunction->getPrintName() << '\n';1432      }1433 1434      // Create an extra entry point if needed. Can also render the target1435      // function ignored if the reference is invalid.1436      handleExternalBranchTarget(Address, *TargetFunction);1437 1438      continue;1439    }1440 1441    // Check if address falls in function padding space - this could be1442    // unmarked data in code. In this case adjust the padding space size.1443    ErrorOr<BinarySection &> Section = getSectionForAddress(Address);1444    assert(Section && "cannot get section for referenced address");1445 1446    if (!Section->isText())1447      continue;1448 1449    // PLT requires special handling and could be ignored in this context.1450    StringRef SectionName = Section->getName();1451    if (SectionName == ".plt" || SectionName == ".plt.got")1452      continue;1453 1454    // Check if it is aarch64 veneer written at Address1455    if (isAArch64() && handleAArch64Veneer(Address))1456      continue;1457 1458    if (opts::processAllFunctions()) {1459      this->errs() << "BOLT-ERROR: cannot process binaries with unmarked "1460                   << "object in code at address 0x"1461                   << Twine::utohexstr(Address) << " belonging to section "1462                   << SectionName << " in current mode\n";1463      exit(1);1464    }1465 1466    TargetFunction = getBinaryFunctionContainingAddress(Address,1467                                                        /*CheckPastEnd=*/false,1468                                                        /*UseMaxSize=*/true);1469    // We are not going to overwrite non-simple functions, but for simple1470    // ones - adjust the padding size.1471    if (TargetFunction && TargetFunction->isSimple()) {1472      this->errs()1473          << "BOLT-WARNING: function " << *TargetFunction1474          << " has an object detected in a padding region at address 0x"1475          << Twine::utohexstr(Address) << '\n';1476      TargetFunction->setMaxSize(TargetFunction->getSize());1477    }1478  }1479 1480  InterproceduralReferences.clear();1481}1482 1483void BinaryContext::postProcessSymbolTable() {1484  fixBinaryDataHoles();1485  bool Valid = true;1486  for (auto &Entry : BinaryDataMap) {1487    BinaryData *BD = Entry.second;1488    if ((BD->getName().starts_with("SYMBOLat") ||1489         BD->getName().starts_with("DATAat")) &&1490        !BD->getParent() && !BD->getSize() && !BD->isAbsolute() &&1491        BD->getSection()) {1492      this->errs() << "BOLT-WARNING: zero-sized top level symbol: " << *BD1493                   << "\n";1494      Valid = false;1495    }1496  }1497  assert(Valid);1498  (void)Valid;1499  generateSymbolHashes();1500}1501 1502void BinaryContext::foldFunction(BinaryFunction &ChildBF,1503                                 BinaryFunction &ParentBF) {1504  assert(!ChildBF.isMultiEntry() && !ParentBF.isMultiEntry() &&1505         "cannot merge functions with multiple entry points");1506 1507  std::unique_lock<llvm::sys::RWMutex> WriteCtxLock(CtxMutex, std::defer_lock);1508  std::unique_lock<llvm::sys::RWMutex> WriteSymbolMapLock(1509      SymbolToFunctionMapMutex, std::defer_lock);1510 1511  const StringRef ChildName = ChildBF.getOneName();1512 1513  // Move symbols over and update bookkeeping info.1514  for (MCSymbol *Symbol : ChildBF.getSymbols()) {1515    ParentBF.getSymbols().push_back(Symbol);1516    WriteSymbolMapLock.lock();1517    SymbolToFunctionMap[Symbol] = &ParentBF;1518    WriteSymbolMapLock.unlock();1519    // NB: there's no need to update BinaryDataMap and GlobalSymbols.1520  }1521  ChildBF.getSymbols().clear();1522 1523  // Reset function mapping for local symbols.1524  for (uint64_t RelOffset : ChildBF.getInternalRefDataRelocations()) {1525    const Relocation *Rel = getRelocationAt(RelOffset);1526    if (!Rel || !Rel->Symbol)1527      continue;1528 1529    WriteSymbolMapLock.lock();1530    SymbolToFunctionMap[Rel->Symbol] = nullptr;1531    WriteSymbolMapLock.unlock();1532  }1533 1534  // Move other names the child function is known under.1535  llvm::move(ChildBF.Aliases, std::back_inserter(ParentBF.Aliases));1536  ChildBF.Aliases.clear();1537 1538  if (HasRelocations) {1539    // Merge execution counts of ChildBF into those of ParentBF.1540    // Without relocations, we cannot reliably merge profiles as both functions1541    // continue to exist and either one can be executed.1542    ChildBF.mergeProfileDataInto(ParentBF);1543 1544    std::shared_lock<llvm::sys::RWMutex> ReadBfsLock(BinaryFunctionsMutex,1545                                                     std::defer_lock);1546    std::unique_lock<llvm::sys::RWMutex> WriteBfsLock(BinaryFunctionsMutex,1547                                                      std::defer_lock);1548    // Remove ChildBF from the global set of functions in relocs mode.1549    ReadBfsLock.lock();1550    auto FI = BinaryFunctions.find(ChildBF.getAddress());1551    ReadBfsLock.unlock();1552 1553    assert(FI != BinaryFunctions.end() && "function not found");1554    assert(&ChildBF == &FI->second && "function mismatch");1555 1556    WriteBfsLock.lock();1557    ChildBF.clearDisasmState();1558    FI = BinaryFunctions.erase(FI);1559    WriteBfsLock.unlock();1560 1561  } else {1562    // In non-relocation mode we keep the function, but rename it.1563    std::string NewName = "__ICF_" + ChildName.str();1564 1565    WriteCtxLock.lock();1566    ChildBF.getSymbols().push_back(Ctx->getOrCreateSymbol(NewName));1567    WriteCtxLock.unlock();1568 1569    ChildBF.setFolded(&ParentBF);1570  }1571 1572  ParentBF.setHasFunctionsFoldedInto();1573}1574 1575void BinaryContext::fixBinaryDataHoles() {1576  assert(validateObjectNesting() && "object nesting inconsistency detected");1577 1578  for (BinarySection &Section : allocatableSections()) {1579    std::vector<std::pair<uint64_t, uint64_t>> Holes;1580 1581    auto isNotHole = [&Section](const binary_data_iterator &Itr) {1582      BinaryData *BD = Itr->second;1583      bool isHole = (!BD->getParent() && !BD->getSize() && BD->isObject() &&1584                     (BD->getName().starts_with("SYMBOLat0x") ||1585                      BD->getName().starts_with("DATAat0x") ||1586                      BD->getName().starts_with("ANONYMOUS")));1587      return !isHole && BD->getSection() == Section && !BD->getParent();1588    };1589 1590    auto BDStart = BinaryDataMap.begin();1591    auto BDEnd = BinaryDataMap.end();1592    auto Itr = FilteredBinaryDataIterator(isNotHole, BDStart, BDEnd);1593    auto End = FilteredBinaryDataIterator(isNotHole, BDEnd, BDEnd);1594 1595    uint64_t EndAddress = Section.getAddress();1596 1597    while (Itr != End) {1598      if (Itr->second->getAddress() > EndAddress) {1599        uint64_t Gap = Itr->second->getAddress() - EndAddress;1600        Holes.emplace_back(EndAddress, Gap);1601      }1602      EndAddress = Itr->second->getEndAddress();1603      ++Itr;1604    }1605 1606    if (EndAddress < Section.getEndAddress())1607      Holes.emplace_back(EndAddress, Section.getEndAddress() - EndAddress);1608 1609    // If there is already a symbol at the start of the hole, grow that symbol1610    // to cover the rest.  Otherwise, create a new symbol to cover the hole.1611    for (std::pair<uint64_t, uint64_t> &Hole : Holes) {1612      BinaryData *BD = getBinaryDataAtAddress(Hole.first);1613      if (BD) {1614        // BD->getSection() can be != Section if there are sections that1615        // overlap.  In this case it is probably safe to just skip the holes1616        // since the overlapping section will not(?) have any symbols in it.1617        if (BD->getSection() == Section)1618          setBinaryDataSize(Hole.first, Hole.second);1619      } else {1620        getOrCreateGlobalSymbol(Hole.first, "HOLEat", Hole.second, 1);1621      }1622    }1623  }1624 1625  assert(validateObjectNesting() && "object nesting inconsistency detected");1626  assert(validateHoles() && "top level hole detected in object map");1627}1628 1629void BinaryContext::printGlobalSymbols(raw_ostream &OS) const {1630  const BinarySection *CurrentSection = nullptr;1631  bool FirstSection = true;1632 1633  for (auto &Entry : BinaryDataMap) {1634    const BinaryData *BD = Entry.second;1635    const BinarySection &Section = BD->getSection();1636    if (FirstSection || Section != *CurrentSection) {1637      uint64_t Address, Size;1638      StringRef Name = Section.getName();1639      if (Section) {1640        Address = Section.getAddress();1641        Size = Section.getSize();1642      } else {1643        Address = BD->getAddress();1644        Size = BD->getSize();1645      }1646      OS << "BOLT-INFO: Section " << Name << ", "1647         << "0x" + Twine::utohexstr(Address) << ":"1648         << "0x" + Twine::utohexstr(Address + Size) << "/" << Size << "\n";1649      CurrentSection = &Section;1650      FirstSection = false;1651    }1652 1653    OS << "BOLT-INFO: ";1654    const BinaryData *P = BD->getParent();1655    while (P) {1656      OS << "  ";1657      P = P->getParent();1658    }1659    OS << *BD << "\n";1660  }1661}1662 1663Expected<unsigned> BinaryContext::getDwarfFile(1664    StringRef Directory, StringRef FileName, unsigned FileNumber,1665    std::optional<MD5::MD5Result> Checksum, std::optional<StringRef> Source,1666    unsigned CUID, unsigned DWARFVersion) {1667  DwarfLineTable &Table = DwarfLineTablesCUMap[CUID];1668  return Table.tryGetFile(Directory, FileName, Checksum, Source, DWARFVersion,1669                          FileNumber);1670}1671 1672unsigned BinaryContext::addDebugFilenameToUnit(const uint32_t DestCUID,1673                                               const uint32_t SrcCUID,1674                                               unsigned FileIndex) {1675  DWARFCompileUnit *SrcUnit = DwCtx->getCompileUnitForOffset(SrcCUID);1676  const DWARFDebugLine::LineTable *LineTable =1677      DwCtx->getLineTableForUnit(SrcUnit);1678  const DWARFDebugLine::FileNameEntry &FileNameEntry =1679      LineTable->Prologue.getFileNameEntry(FileIndex);1680  // Dir indexes start at 1 and a dir index 01681  // means empty dir.1682  StringRef Dir = "";1683  if (FileNameEntry.DirIdx != 0) {1684    if (std::optional<const char *> DirName = dwarf::toString(1685            LineTable->Prologue.IncludeDirectories[FileNameEntry.DirIdx - 1])) {1686      Dir = *DirName;1687    }1688  }1689  StringRef FileName = "";1690  if (std::optional<const char *> FName = dwarf::toString(FileNameEntry.Name))1691    FileName = *FName;1692  assert(FileName != "");1693  DWARFCompileUnit *DstUnit = DwCtx->getCompileUnitForOffset(DestCUID);1694  return cantFail(getDwarfFile(Dir, FileName, 0, std::nullopt, std::nullopt,1695                               DestCUID, DstUnit->getVersion()));1696}1697 1698std::vector<BinaryFunction *> BinaryContext::getSortedFunctions() {1699  std::vector<BinaryFunction *> SortedFunctions(BinaryFunctions.size());1700  llvm::transform(llvm::make_second_range(BinaryFunctions),1701                  SortedFunctions.begin(),1702                  [](BinaryFunction &BF) { return &BF; });1703 1704  llvm::stable_sort(SortedFunctions, compareBinaryFunctionByIndex);1705  return SortedFunctions;1706}1707 1708std::vector<BinaryFunction *> BinaryContext::getAllBinaryFunctions() {1709  std::vector<BinaryFunction *> AllFunctions;1710  AllFunctions.reserve(BinaryFunctions.size() + InjectedBinaryFunctions.size());1711  llvm::transform(llvm::make_second_range(BinaryFunctions),1712                  std::back_inserter(AllFunctions),1713                  [](BinaryFunction &BF) { return &BF; });1714  llvm::copy(InjectedBinaryFunctions, std::back_inserter(AllFunctions));1715 1716  return AllFunctions;1717}1718 1719std::optional<DWARFUnit *> BinaryContext::getDWOCU(uint64_t DWOId) {1720  auto Iter = DWOCUs.find(DWOId);1721  if (Iter == DWOCUs.end())1722    return std::nullopt;1723 1724  return Iter->second;1725}1726 1727DWARFContext *BinaryContext::getDWOContext() const {1728  if (DWOCUs.empty())1729    return nullptr;1730  return &DWOCUs.begin()->second->getContext();1731}1732 1733bool BinaryContext::isValidDwarfUnit(DWARFUnit &DU) const {1734  // Invalid DWARF unit with a DWOId but lacking a dwo_name.1735  if (DU.getDWOId() && !DU.isDWOUnit() &&1736      !DU.getUnitDIE().find(1737          {dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name})) {1738    this->outs() << "BOLT-ERROR: broken DWARF found in CU at offset 0x"1739                 << Twine::utohexstr(DU.getOffset()) << " (DWOId=0x"1740                 << Twine::utohexstr(*(DU.getDWOId()))1741                 << ", missing DW_AT_dwo_name / DW_AT_GNU_dwo_name)\n";1742    return false;1743  }1744  return true;1745}1746 1747/// Handles DWO sections that can either be in .o, .dwo or .dwp files.1748void BinaryContext::preprocessDWODebugInfo() {1749  for (const std::unique_ptr<DWARFUnit> &CU : DwCtx->compile_units()) {1750    DWARFUnit *const DwarfUnit = CU.get();1751    if (!isValidDwarfUnit(*DwarfUnit))1752      continue;1753    if (std::optional<uint64_t> DWOId = DwarfUnit->getDWOId()) {1754      std::string DWOName = dwarf::toString(1755          DwarfUnit->getUnitDIE().find(1756              {dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}),1757          "");1758      SmallString<16> AbsolutePath(DWOName);1759      std::string DWOCompDir = DwarfUnit->getCompilationDir();1760      if (!opts::CompDirOverride.empty()) {1761        DWOCompDir = opts::CompDirOverride;1762      } else if (!sys::fs::exists(DWOCompDir) && sys::fs::exists(DWOName)) {1763        DWOCompDir = ".";1764        this->outs()1765            << "BOLT-WARNING: Debug Fission: Debug Compilation Directory of "1766            << DWOName1767            << " does not exist. Relative path will be used to process .dwo "1768               "files.\n";1769      }1770      // Prevent failures when DWOName is already an absolute path.1771      sys::path::make_absolute(DWOCompDir, AbsolutePath);1772      DWARFUnit *DWOCU =1773          DwarfUnit->getNonSkeletonUnitDIE(false, AbsolutePath).getDwarfUnit();1774      if (!DWOCU->isDWOUnit()) {1775        this->outs()1776            << "BOLT-WARNING: Debug Fission: DWO debug information for "1777            << DWOName1778            << " was not retrieved and won't be updated. Please check "1779               "relative path or use '--comp-dir-override' to specify the base "1780               "location.\n";1781        continue;1782      }1783      DWOCUs[*DWOId] = DWOCU;1784    }1785  }1786  if (!DWOCUs.empty())1787    this->outs() << "BOLT-INFO: processing split DWARF\n";1788}1789 1790void BinaryContext::preprocessDebugInfo() {1791  struct CURange {1792    uint64_t LowPC;1793    uint64_t HighPC;1794    DWARFUnit *Unit;1795 1796    bool operator<(const CURange &Other) const { return LowPC < Other.LowPC; }1797  };1798 1799  // Building a map of address ranges to CUs similar to .debug_aranges and use1800  // it to assign CU to functions.1801  std::vector<CURange> AllRanges;1802  AllRanges.reserve(DwCtx->getNumCompileUnits());1803  for (const std::unique_ptr<DWARFUnit> &CU : DwCtx->compile_units()) {1804    Expected<DWARFAddressRangesVector> RangesOrError =1805        CU->getUnitDIE().getAddressRanges();1806    if (!RangesOrError) {1807      consumeError(RangesOrError.takeError());1808      continue;1809    }1810    for (DWARFAddressRange &Range : *RangesOrError) {1811      // Parts of the debug info could be invalidated due to corresponding code1812      // being removed from the binary by the linker. Hence we check if the1813      // address is a valid one.1814      if (containsAddress(Range.LowPC))1815        AllRanges.emplace_back(CURange{Range.LowPC, Range.HighPC, CU.get()});1816    }1817 1818    ContainsDwarf5 |= CU->getVersion() >= 5;1819    ContainsDwarfLegacy |= CU->getVersion() < 5;1820  }1821 1822  llvm::sort(AllRanges);1823  for (auto &KV : BinaryFunctions) {1824    const uint64_t FunctionAddress = KV.first;1825    BinaryFunction &Function = KV.second;1826 1827    auto It = llvm::partition_point(1828        AllRanges, [=](CURange R) { return R.HighPC <= FunctionAddress; });1829    if (It == AllRanges.end() || It->LowPC > FunctionAddress) {1830      continue;1831    }1832    Function.addDWARFUnit(It->Unit);1833 1834    // Go forward and add all units from ranges that cover the function.1835    while (++It != AllRanges.end()) {1836      if (It->LowPC > FunctionAddress || FunctionAddress >= It->HighPC)1837        break;1838      Function.addDWARFUnit(It->Unit);1839    }1840  }1841 1842  // Discover units with debug info that needs to be updated.1843  for (const auto &KV : BinaryFunctions) {1844    const BinaryFunction &BF = KV.second;1845    if (shouldEmit(BF) && !BF.getDWARFUnits().empty())1846      for (const auto &[_, Unit] : BF.getDWARFUnits())1847        ProcessedCUs.insert(Unit);1848  }1849  // Clear debug info for functions from units that we are not going to process.1850  for (auto &KV : BinaryFunctions) {1851    BinaryFunction &BF = KV.second;1852    // Collect units to remove to avoid iterator invalidation1853    SmallVector<DWARFUnit *, 1> UnitsToRemove;1854    for (const auto &[_, Unit] : BF.getDWARFUnits()) {1855      if (!ProcessedCUs.count(Unit))1856        UnitsToRemove.push_back(Unit);1857    }1858    // Remove the collected units1859    for (auto *Unit : UnitsToRemove) {1860      BF.removeDWARFUnit(Unit);1861    }1862  }1863 1864  if (opts::Verbosity >= 1) {1865    this->outs() << "BOLT-INFO: " << ProcessedCUs.size() << " out of "1866                 << DwCtx->getNumCompileUnits() << " CUs will be updated\n";1867  }1868 1869  preprocessDWODebugInfo();1870 1871  // Populate MCContext with DWARF files from all units.1872  StringRef GlobalPrefix = AsmInfo->getPrivateGlobalPrefix();1873  for (const std::unique_ptr<DWARFUnit> &CU : DwCtx->compile_units()) {1874    const uint64_t CUID = CU->getOffset();1875    DwarfLineTable &BinaryLineTable = getDwarfLineTable(CUID);1876    BinaryLineTable.setLabel(Ctx->getOrCreateSymbol(1877        GlobalPrefix + "line_table_start" + Twine(CUID)));1878 1879    if (!ProcessedCUs.count(CU.get()))1880      continue;1881 1882    const DWARFDebugLine::LineTable *LineTable =1883        DwCtx->getLineTableForUnit(CU.get());1884    const std::vector<DWARFDebugLine::FileNameEntry> &FileNames =1885        LineTable->Prologue.FileNames;1886 1887    uint16_t DwarfVersion = LineTable->Prologue.getVersion();1888    if (DwarfVersion >= 5) {1889      std::optional<MD5::MD5Result> Checksum;1890      if (LineTable->Prologue.ContentTypes.HasMD5)1891        Checksum = LineTable->Prologue.FileNames[0].Checksum;1892      std::optional<const char *> Name =1893          dwarf::toString(CU->getUnitDIE().find(dwarf::DW_AT_name), nullptr);1894      if (std::optional<uint64_t> DWOID = CU->getDWOId()) {1895        auto Iter = DWOCUs.find(*DWOID);1896        if (Iter == DWOCUs.end()) {1897          this->errs() << "BOLT-ERROR: DWO CU was not found for " << Name1898                       << '\n';1899          exit(1);1900        }1901        Name = dwarf::toString(1902            Iter->second->getUnitDIE().find(dwarf::DW_AT_name), nullptr);1903      }1904      BinaryLineTable.setRootFile(CU->getCompilationDir(), *Name, Checksum,1905                                  std::nullopt);1906    }1907 1908    BinaryLineTable.setDwarfVersion(DwarfVersion);1909 1910    // Assign a unique label to every line table, one per CU.1911    // Make sure empty debug line tables are registered too.1912    if (FileNames.empty()) {1913      cantFail(getDwarfFile("", "<unknown>", 0, std::nullopt, std::nullopt,1914                            CUID, DwarfVersion));1915      continue;1916    }1917    const uint32_t Offset = DwarfVersion < 5 ? 1 : 0;1918    for (size_t I = 0, Size = FileNames.size(); I != Size; ++I) {1919      // Dir indexes start at 1, as DWARF file numbers, and a dir index 01920      // means empty dir.1921      StringRef Dir = "";1922      if (FileNames[I].DirIdx != 0 || DwarfVersion >= 5)1923        if (std::optional<const char *> DirName = dwarf::toString(1924                LineTable->Prologue1925                    .IncludeDirectories[FileNames[I].DirIdx - Offset]))1926          Dir = *DirName;1927      StringRef FileName = "";1928      if (std::optional<const char *> FName =1929              dwarf::toString(FileNames[I].Name))1930        FileName = *FName;1931      assert(FileName != "");1932      std::optional<MD5::MD5Result> Checksum;1933      if (DwarfVersion >= 5 && LineTable->Prologue.ContentTypes.HasMD5)1934        Checksum = LineTable->Prologue.FileNames[I].Checksum;1935      cantFail(getDwarfFile(Dir, FileName, 0, Checksum, std::nullopt, CUID,1936                            DwarfVersion));1937    }1938  }1939}1940 1941bool BinaryContext::shouldEmit(const BinaryFunction &Function) const {1942  if (Function.isPseudo())1943    return false;1944 1945  if (opts::processAllFunctions())1946    return true;1947 1948  if (Function.isIgnored())1949    return false;1950 1951  // In relocation mode we will emit non-simple functions with CFG.1952  // If the function does not have a CFG it should be marked as ignored.1953  return HasRelocations || Function.isSimple();1954}1955 1956void BinaryContext::dump(const MCInst &Inst) const {1957  if (LLVM_UNLIKELY(!InstPrinter)) {1958    dbgs() << "Cannot dump for InstPrinter is not initialized.\n";1959    return;1960  }1961  InstPrinter->printInst(&Inst, 0, "", *STI, dbgs());1962  dbgs() << "\n";1963}1964 1965void BinaryContext::printCFI(raw_ostream &OS, const MCCFIInstruction &Inst) {1966  uint32_t Operation = Inst.getOperation();1967  switch (Operation) {1968  case MCCFIInstruction::OpSameValue:1969    OS << "OpSameValue Reg" << Inst.getRegister();1970    break;1971  case MCCFIInstruction::OpRememberState:1972    OS << "OpRememberState";1973    break;1974  case MCCFIInstruction::OpRestoreState:1975    OS << "OpRestoreState";1976    break;1977  case MCCFIInstruction::OpOffset:1978    OS << "OpOffset Reg" << Inst.getRegister() << " " << Inst.getOffset();1979    break;1980  case MCCFIInstruction::OpDefCfaRegister:1981    OS << "OpDefCfaRegister Reg" << Inst.getRegister();1982    break;1983  case MCCFIInstruction::OpDefCfaOffset:1984    OS << "OpDefCfaOffset " << Inst.getOffset();1985    break;1986  case MCCFIInstruction::OpDefCfa:1987    OS << "OpDefCfa Reg" << Inst.getRegister() << " " << Inst.getOffset();1988    break;1989  case MCCFIInstruction::OpRelOffset:1990    OS << "OpRelOffset Reg" << Inst.getRegister() << " " << Inst.getOffset();1991    break;1992  case MCCFIInstruction::OpAdjustCfaOffset:1993    OS << "OfAdjustCfaOffset " << Inst.getOffset();1994    break;1995  case MCCFIInstruction::OpEscape:1996    OS << "OpEscape";1997    break;1998  case MCCFIInstruction::OpRestore:1999    OS << "OpRestore Reg" << Inst.getRegister();2000    break;2001  case MCCFIInstruction::OpUndefined:2002    OS << "OpUndefined Reg" << Inst.getRegister();2003    break;2004  case MCCFIInstruction::OpRegister:2005    OS << "OpRegister Reg" << Inst.getRegister() << " Reg"2006       << Inst.getRegister2();2007    break;2008  case MCCFIInstruction::OpWindowSave:2009    OS << "OpWindowSave";2010    break;2011  case MCCFIInstruction::OpGnuArgsSize:2012    OS << "OpGnuArgsSize";2013    break;2014  case MCCFIInstruction::OpNegateRAState:2015    OS << "OpNegateRAState";2016    break;2017  default:2018    OS << "Op#" << Operation;2019    break;2020  }2021}2022 2023MarkerSymType BinaryContext::getMarkerType(const SymbolRef &Symbol) const {2024  // For aarch64 and riscv, the ABI defines mapping symbols so we identify data2025  // in the code section (see IHI0056B). $x identifies a symbol starting code or2026  // the end of a data chunk inside code, $d identifies start of data.2027  if (isX86() || ELFSymbolRef(Symbol).getSize())2028    return MarkerSymType::NONE;2029 2030  Expected<StringRef> NameOrError = Symbol.getName();2031  Expected<object::SymbolRef::Type> TypeOrError = Symbol.getType();2032 2033  if (!TypeOrError || !NameOrError)2034    return MarkerSymType::NONE;2035 2036  if (*TypeOrError != SymbolRef::ST_Unknown)2037    return MarkerSymType::NONE;2038 2039  if (*NameOrError == "$x" || NameOrError->starts_with("$x."))2040    return MarkerSymType::CODE;2041 2042  // $x<ISA>2043  if (isRISCV() && NameOrError->starts_with("$x"))2044    return MarkerSymType::CODE;2045 2046  if (*NameOrError == "$d" || NameOrError->starts_with("$d."))2047    return MarkerSymType::DATA;2048 2049  return MarkerSymType::NONE;2050}2051 2052bool BinaryContext::isMarker(const SymbolRef &Symbol) const {2053  return getMarkerType(Symbol) != MarkerSymType::NONE;2054}2055 2056static void printDebugInfo(raw_ostream &OS, const MCInst &Instruction,2057                           const BinaryFunction *Function,2058                           DWARFContext *DwCtx) {2059  const ClusteredRows *LineTableRows =2060      ClusteredRows::fromSMLoc(Instruction.getLoc());2061  if (LineTableRows == nullptr)2062    return;2063 2064  // File name and line number should be the same for all CUs.2065  // So it is sufficient to check the first one.2066  DebugLineTableRowRef RowRef = LineTableRows->getRows().front();2067  const DWARFDebugLine::LineTable *LineTable = DwCtx->getLineTableForUnit(2068      DwCtx->getCompileUnitForOffset(RowRef.DwCompileUnitIndex));2069 2070  if (!LineTable)2071    return;2072 2073  const DWARFDebugLine::Row &Row = LineTable->Rows[RowRef.RowIndex - 1];2074  StringRef FileName = "";2075 2076  if (std::optional<const char *> FName =2077          dwarf::toString(LineTable->Prologue.getFileNameEntry(Row.File).Name))2078    FileName = *FName;2079  OS << " # debug line " << FileName << ":" << Row.Line;2080  if (Row.Column)2081    OS << ":" << Row.Column;2082  if (Row.Discriminator)2083    OS << " discriminator:" << Row.Discriminator;2084}2085 2086ArrayRef<uint8_t> BinaryContext::extractData(uint64_t Address,2087                                             uint64_t Size) const {2088  ArrayRef<uint8_t> Res;2089 2090  const ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);2091  if (!Section || Section->isVirtual())2092    return Res;2093 2094  if (!Section->containsRange(Address, Size))2095    return Res;2096 2097  auto *Bytes =2098      reinterpret_cast<const uint8_t *>(Section->getContents().data());2099  return ArrayRef<uint8_t>(Bytes + Address - Section->getAddress(), Size);2100}2101 2102void BinaryContext::printData(raw_ostream &OS, ArrayRef<uint8_t> Data,2103                              uint64_t Offset) const {2104  DataExtractor DE(Data, AsmInfo->isLittleEndian(),2105                   AsmInfo->getCodePointerSize());2106  uint64_t DataOffset = 0;2107  while (DataOffset + 4 <= Data.size()) {2108    OS << format("    %08" PRIx64 ": \t.word\t0x", Offset + DataOffset);2109    const auto Word = DE.getUnsigned(&DataOffset, 4);2110    OS << Twine::utohexstr(Word) << '\n';2111  }2112  if (DataOffset + 2 <= Data.size()) {2113    OS << format("    %08" PRIx64 ": \t.short\t0x", Offset + DataOffset);2114    const auto Short = DE.getUnsigned(&DataOffset, 2);2115    OS << Twine::utohexstr(Short) << '\n';2116  }2117  if (DataOffset + 1 == Data.size()) {2118    OS << format("    %08" PRIx64 ": \t.byte\t0x%x\n", Offset + DataOffset,2119                 Data[DataOffset]);2120  }2121}2122 2123void BinaryContext::printInstruction(raw_ostream &OS, const MCInst &Instruction,2124                                     uint64_t Offset,2125                                     const BinaryFunction *Function,2126                                     bool PrintMCInst, bool PrintMemData,2127                                     bool PrintRelocations,2128                                     StringRef Endl) const {2129  OS << format("    %08" PRIx64 ": ", Offset);2130  if (MIB->isCFI(Instruction)) {2131    uint32_t Offset = Instruction.getOperand(0).getImm();2132    OS << "\t!CFI\t$" << Offset << "\t; ";2133    if (Function)2134      printCFI(OS, *Function->getCFIFor(Instruction));2135    OS << Endl;2136    return;2137  }2138  if (std::optional<uint32_t> DynamicID =2139          MIB->getDynamicBranchID(Instruction)) {2140    OS << "\tjit\t" << MIB->getTargetSymbol(Instruction)->getName()2141       << " # ID: " << DynamicID;2142  } else {2143    // If there are annotations on the instruction, the MCInstPrinter will fail2144    // to print the preferred alias as it only does so when the number of2145    // operands is as expected. See2146    // https://github.com/llvm/llvm-project/blob/782f1a0d895646c364a53f9dcdd6d4ec1f3e5ea0/llvm/lib/MC/MCInstPrinter.cpp#L1422147    // Therefore, create a temporary copy of the Inst from which the annotations2148    // are removed, and print that Inst.2149    MCInst InstNoAnnot = Instruction;2150    MIB->stripAnnotations(InstNoAnnot);2151    InstPrinter->printInst(&InstNoAnnot, 0, "", *STI, OS);2152  }2153  if (MIB->isCall(Instruction)) {2154    if (MIB->isTailCall(Instruction))2155      OS << " # TAILCALL ";2156    if (MIB->isInvoke(Instruction)) {2157      const std::optional<MCPlus::MCLandingPad> EHInfo =2158          MIB->getEHInfo(Instruction);2159      OS << " # handler: ";2160      if (EHInfo->first)2161        OS << *EHInfo->first;2162      else2163        OS << '0';2164      OS << "; action: " << EHInfo->second;2165      const int64_t GnuArgsSize = MIB->getGnuArgsSize(Instruction);2166      if (GnuArgsSize >= 0)2167        OS << "; GNU_args_size = " << GnuArgsSize;2168    }2169  } else if (MIB->isIndirectBranch(Instruction)) {2170    if (uint64_t JTAddress = MIB->getJumpTable(Instruction)) {2171      OS << " # JUMPTABLE @0x" << Twine::utohexstr(JTAddress);2172    } else {2173      OS << " # UNKNOWN CONTROL FLOW";2174    }2175  }2176  if (std::optional<uint32_t> Offset = MIB->getOffset(Instruction))2177    OS << " # Offset: " << *Offset;2178  if (std::optional<uint32_t> Size = MIB->getSize(Instruction))2179    OS << " # Size: " << *Size;2180  if (MCSymbol *Label = MIB->getInstLabel(Instruction))2181    OS << " # Label: " << *Label;2182 2183  MIB->printAnnotations(Instruction, OS, PrintMemData || opts::PrintMemData);2184 2185  if (opts::PrintDebugInfo)2186    printDebugInfo(OS, Instruction, Function, DwCtx.get());2187 2188  if ((opts::PrintRelocations || PrintRelocations) && Function) {2189    const uint64_t Size = computeCodeSize(&Instruction, &Instruction + 1);2190    Function->printRelocations(OS, Offset, Size);2191  }2192 2193  OS << Endl;2194 2195  if (PrintMCInst) {2196    Instruction.dump_pretty(OS, InstPrinter.get());2197    OS << Endl;2198  }2199}2200 2201std::optional<uint64_t>2202BinaryContext::getBaseAddressForMapping(uint64_t MMapAddress,2203                                        uint64_t FileOffset) const {2204  // Find a segment with a matching file offset.2205  for (auto &KV : SegmentMapInfo) {2206    const SegmentInfo &SegInfo = KV.second;2207    // Only consider executable segments.2208    if (!SegInfo.IsExecutable)2209      continue;2210    // FileOffset is got from perf event,2211    // and it is equal to alignDown(SegInfo.FileOffset, pagesize).2212    // If the pagesize is not equal to SegInfo.Alignment.2213    // FileOffset and SegInfo.FileOffset should be aligned first,2214    // and then judge whether they are equal.2215    if (alignDown(SegInfo.FileOffset, SegInfo.Alignment) ==2216        alignDown(FileOffset, SegInfo.Alignment)) {2217      // The function's offset from base address in VAS is aligned by pagesize2218      // instead of SegInfo.Alignment. Pagesize can't be got from perf events.2219      // However, The ELF document says that SegInfo.FileOffset should equal2220      // to SegInfo.Address, modulo the pagesize.2221      // Reference: https://refspecs.linuxfoundation.org/elf/elf.pdf2222 2223      // So alignDown(SegInfo.Address, pagesize) can be calculated by:2224      // alignDown(SegInfo.Address, pagesize)2225      //   = SegInfo.Address - (SegInfo.Address % pagesize)2226      //   = SegInfo.Address - (SegInfo.FileOffset % pagesize)2227      //   = SegInfo.Address - SegInfo.FileOffset +2228      //     alignDown(SegInfo.FileOffset, pagesize)2229      //   = SegInfo.Address - SegInfo.FileOffset + FileOffset2230      return MMapAddress - (SegInfo.Address - SegInfo.FileOffset + FileOffset);2231    }2232  }2233 2234  return std::nullopt;2235}2236 2237ErrorOr<BinarySection &> BinaryContext::getSectionForAddress(uint64_t Address) {2238  auto SI = AddressToSection.upper_bound(Address);2239  if (SI != AddressToSection.begin()) {2240    --SI;2241    uint64_t UpperBound = SI->first + SI->second->getSize();2242    if (!SI->second->getSize())2243      UpperBound += 1;2244    if (UpperBound > Address)2245      return *SI->second;2246  }2247  return std::make_error_code(std::errc::bad_address);2248}2249 2250ErrorOr<StringRef>2251BinaryContext::getSectionNameForAddress(uint64_t Address) const {2252  if (ErrorOr<const BinarySection &> Section = getSectionForAddress(Address))2253    return Section->getName();2254  return std::make_error_code(std::errc::bad_address);2255}2256 2257BinarySection &BinaryContext::registerSection(BinarySection *Section) {2258  auto Res = Sections.insert(Section);2259  (void)Res;2260  assert(Res.second && "can't register the same section twice.");2261 2262  // Only register allocatable sections in the AddressToSection map.2263  if (Section->isAllocatable() && Section->getAddress())2264    AddressToSection.insert(std::make_pair(Section->getAddress(), Section));2265  NameToSection.insert(2266      std::make_pair(std::string(Section->getName()), Section));2267  if (Section->hasSectionRef())2268    SectionRefToBinarySection.insert(2269        std::make_pair(Section->getSectionRef(), Section));2270 2271  LLVM_DEBUG(dbgs() << "BOLT-DEBUG: registering " << *Section << "\n");2272  return *Section;2273}2274 2275BinarySection &BinaryContext::registerSection(SectionRef Section) {2276  return registerSection(new BinarySection(*this, Section));2277}2278 2279BinarySection &2280BinaryContext::registerSection(const Twine &SectionName,2281                               const BinarySection &OriginalSection) {2282  return registerSection(2283      new BinarySection(*this, SectionName, OriginalSection));2284}2285 2286BinarySection &2287BinaryContext::registerOrUpdateSection(const Twine &Name, unsigned ELFType,2288                                       unsigned ELFFlags, uint8_t *Data,2289                                       uint64_t Size, unsigned Alignment) {2290  auto NamedSections = getSectionByName(Name);2291  if (NamedSections.begin() != NamedSections.end()) {2292    assert(std::next(NamedSections.begin()) == NamedSections.end() &&2293           "can only update unique sections");2294    BinarySection *Section = NamedSections.begin()->second;2295 2296    LLVM_DEBUG(dbgs() << "BOLT-DEBUG: updating " << *Section << " -> ");2297    const bool Flag = Section->isAllocatable();2298    (void)Flag;2299    Section->update(Data, Size, Alignment, ELFType, ELFFlags);2300    LLVM_DEBUG(dbgs() << *Section << "\n");2301    // FIXME: Fix section flags/attributes for MachO.2302    if (isELF())2303      assert(Flag == Section->isAllocatable() &&2304             "can't change section allocation status");2305    return *Section;2306  }2307 2308  return registerSection(2309      new BinarySection(*this, Name, Data, Size, Alignment, ELFType, ELFFlags));2310}2311 2312void BinaryContext::deregisterSectionName(const BinarySection &Section) {2313  auto NameRange = NameToSection.equal_range(Section.getName().str());2314  while (NameRange.first != NameRange.second) {2315    if (NameRange.first->second == &Section) {2316      NameToSection.erase(NameRange.first);2317      break;2318    }2319    ++NameRange.first;2320  }2321}2322 2323void BinaryContext::deregisterUnusedSections() {2324  ErrorOr<BinarySection &> AbsSection = getUniqueSectionByName("<absolute>");2325  for (auto SI = Sections.begin(); SI != Sections.end();) {2326    BinarySection *Section = *SI;2327    // We check getOutputData() instead of getOutputSize() because sometimes2328    // zero-sized .text.cold sections are allocated.2329    if (Section->hasSectionRef() || Section->getOutputData() ||2330        (AbsSection && Section == &AbsSection.get())) {2331      ++SI;2332      continue;2333    }2334 2335    LLVM_DEBUG(dbgs() << "LLVM-DEBUG: deregistering " << Section->getName()2336                      << '\n';);2337    deregisterSectionName(*Section);2338    SI = Sections.erase(SI);2339    delete Section;2340  }2341}2342 2343bool BinaryContext::deregisterSection(BinarySection &Section) {2344  BinarySection *SectionPtr = &Section;2345  auto Itr = Sections.find(SectionPtr);2346  if (Itr != Sections.end()) {2347    auto Range = AddressToSection.equal_range(SectionPtr->getAddress());2348    while (Range.first != Range.second) {2349      if (Range.first->second == SectionPtr) {2350        AddressToSection.erase(Range.first);2351        break;2352      }2353      ++Range.first;2354    }2355 2356    deregisterSectionName(*SectionPtr);2357    Sections.erase(Itr);2358    delete SectionPtr;2359    return true;2360  }2361  return false;2362}2363 2364void BinaryContext::renameSection(BinarySection &Section,2365                                  const Twine &NewName) {2366  auto Itr = Sections.find(&Section);2367  assert(Itr != Sections.end() && "Section must exist to be renamed.");2368  Sections.erase(Itr);2369 2370  deregisterSectionName(Section);2371 2372  Section.Name = NewName.str();2373  Section.setOutputName(Section.Name);2374 2375  NameToSection.insert(std::make_pair(Section.Name, &Section));2376 2377  // Reinsert with the new name.2378  Sections.insert(&Section);2379}2380 2381void BinaryContext::printSections(raw_ostream &OS) const {2382  for (BinarySection *const &Section : Sections)2383    OS << "BOLT-INFO: " << *Section << "\n";2384}2385 2386BinarySection &BinaryContext::absoluteSection() {2387  if (ErrorOr<BinarySection &> Section = getUniqueSectionByName("<absolute>"))2388    return *Section;2389  return registerOrUpdateSection("<absolute>", ELF::SHT_NULL, 0u);2390}2391 2392ErrorOr<uint64_t> BinaryContext::getUnsignedValueAtAddress(uint64_t Address,2393                                                           size_t Size) const {2394  const ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);2395  if (!Section)2396    return std::make_error_code(std::errc::bad_address);2397 2398  if (Section->isVirtual())2399    return 0;2400 2401  DataExtractor DE(Section->getContents(), AsmInfo->isLittleEndian(),2402                   AsmInfo->getCodePointerSize());2403  auto ValueOffset = static_cast<uint64_t>(Address - Section->getAddress());2404  return DE.getUnsigned(&ValueOffset, Size);2405}2406 2407ErrorOr<int64_t> BinaryContext::getSignedValueAtAddress(uint64_t Address,2408                                                        size_t Size) const {2409  const ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);2410  if (!Section)2411    return std::make_error_code(std::errc::bad_address);2412 2413  if (Section->isVirtual())2414    return 0;2415 2416  DataExtractor DE(Section->getContents(), AsmInfo->isLittleEndian(),2417                   AsmInfo->getCodePointerSize());2418  auto ValueOffset = static_cast<uint64_t>(Address - Section->getAddress());2419  return DE.getSigned(&ValueOffset, Size);2420}2421 2422void BinaryContext::addRelocation(uint64_t Address, MCSymbol *Symbol,2423                                  uint32_t Type, uint64_t Addend,2424                                  uint64_t Value) {2425  ErrorOr<BinarySection &> Section = getSectionForAddress(Address);2426  assert(Section && "cannot find section for address");2427  Section->addRelocation(Address - Section->getAddress(), Symbol, Type, Addend,2428                         Value);2429}2430 2431void BinaryContext::addDynamicRelocation(uint64_t Address, MCSymbol *Symbol,2432                                         uint32_t Type, uint64_t Addend,2433                                         uint64_t Value) {2434  ErrorOr<BinarySection &> Section = getSectionForAddress(Address);2435  assert(Section && "cannot find section for address");2436  Section->addDynamicRelocation(Address - Section->getAddress(), Symbol, Type,2437                                Addend, Value);2438}2439 2440bool BinaryContext::removeRelocationAt(uint64_t Address) {2441  ErrorOr<BinarySection &> Section = getSectionForAddress(Address);2442  assert(Section && "cannot find section for address");2443  return Section->removeRelocationAt(Address - Section->getAddress());2444}2445 2446const Relocation *BinaryContext::getRelocationAt(uint64_t Address) const {2447  ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);2448  if (!Section)2449    return nullptr;2450 2451  return Section->getRelocationAt(Address - Section->getAddress());2452}2453 2454const Relocation *2455BinaryContext::getDynamicRelocationAt(uint64_t Address) const {2456  ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);2457  if (!Section)2458    return nullptr;2459 2460  return Section->getDynamicRelocationAt(Address - Section->getAddress());2461}2462 2463void BinaryContext::markAmbiguousRelocations(BinaryData &BD,2464                                             const uint64_t Address) {2465  auto setImmovable = [&](BinaryData &BD) {2466    BinaryData *Root = BD.getAtomicRoot();2467    LLVM_DEBUG(if (Root->isMoveable()) {2468      dbgs() << "BOLT-DEBUG: setting " << *Root << " as immovable "2469             << "due to ambiguous relocation referencing 0x"2470             << Twine::utohexstr(Address) << '\n';2471    });2472    Root->setIsMoveable(false);2473  };2474 2475  if (Address == BD.getAddress()) {2476    setImmovable(BD);2477 2478    // Set previous symbol as immovable2479    BinaryData *Prev = getBinaryDataContainingAddress(Address - 1);2480    if (Prev && Prev->getEndAddress() == BD.getAddress())2481      setImmovable(*Prev);2482  }2483 2484  if (Address == BD.getEndAddress()) {2485    setImmovable(BD);2486 2487    // Set next symbol as immovable2488    BinaryData *Next = getBinaryDataContainingAddress(BD.getEndAddress());2489    if (Next && Next->getAddress() == BD.getEndAddress())2490      setImmovable(*Next);2491  }2492}2493 2494BinaryFunction *BinaryContext::getFunctionForSymbol(const MCSymbol *Symbol,2495                                                    uint64_t *EntryDesc) {2496  std::shared_lock<llvm::sys::RWMutex> Lock(SymbolToFunctionMapMutex);2497  auto BFI = SymbolToFunctionMap.find(Symbol);2498  if (BFI == SymbolToFunctionMap.end())2499    return nullptr;2500 2501  BinaryFunction *BF = BFI->second;2502  if (EntryDesc)2503    *EntryDesc = BF->getEntryIDForSymbol(Symbol);2504 2505  return BF;2506}2507 2508std::string2509BinaryContext::generateBugReportMessage(StringRef Message,2510                                        const BinaryFunction &Function) const {2511  std::string Msg;2512  raw_string_ostream SS(Msg);2513  SS << "=======================================\n";2514  SS << "BOLT is unable to proceed because it couldn't properly understand "2515        "this function.\n";2516  SS << "If you are running the most recent version of BOLT, you may "2517        "want to "2518        "report this and paste this dump.\nPlease check that there is no "2519        "sensitive contents being shared in this dump.\n";2520  SS << "\nOffending function: " << Function.getPrintName() << "\n\n";2521  ScopedPrinter SP(SS);2522  SP.printBinaryBlock("Function contents", *Function.getData());2523  SS << "\n";2524  const_cast<BinaryFunction &>(Function).print(SS, "");2525  SS << "ERROR: " << Message;2526  SS << "\n=======================================\n";2527  return Msg;2528}2529 2530BinaryFunction *2531BinaryContext::createInjectedBinaryFunction(const std::string &Name,2532                                            bool IsSimple) {2533  InjectedBinaryFunctions.push_back(new BinaryFunction(Name, *this, IsSimple));2534  BinaryFunction *BF = InjectedBinaryFunctions.back();2535  setSymbolToFunctionMap(BF->getSymbol(), BF);2536  BF->CurrentState = BinaryFunction::State::CFG;2537  return BF;2538}2539 2540BinaryFunction *2541BinaryContext::createInstructionPatch(uint64_t Address,2542                                      const InstructionListType &Instructions,2543                                      const Twine &Name) {2544  ErrorOr<BinarySection &> Section = getSectionForAddress(Address);2545  assert(Section && "cannot get section for patching");2546  assert(Section->hasSectionRef() && Section->isText() &&2547         "can only patch input file code sections");2548 2549  const uint64_t FileOffset =2550      Section->getInputFileOffset() + Address - Section->getAddress();2551 2552  std::string PatchName = Name.str();2553  if (PatchName.empty()) {2554    // Assign unique name to the patch.2555    static uint64_t N = 0;2556    PatchName = "__BP_" + std::to_string(N++);2557  }2558 2559  BinaryFunction *PBF = createInjectedBinaryFunction(PatchName);2560  PBF->setOutputAddress(Address);2561  PBF->setFileOffset(FileOffset);2562  PBF->setOriginSection(&Section.get());2563  PBF->addBasicBlock()->addInstructions(Instructions);2564  PBF->setIsPatch(true);2565 2566  // Don't create symbol table entry if the name wasn't specified.2567  if (Name.str().empty())2568    PBF->setAnonymous(true);2569 2570  return PBF;2571}2572 2573std::pair<size_t, size_t>2574BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {2575  // Use the original size for non-simple functions.2576  if (!BF.isSimple() || BF.isIgnored())2577    return std::make_pair(BF.getSize(), 0);2578 2579  // Adjust branch instruction to match the current layout.2580  if (FixBranches)2581    BF.fixBranches();2582 2583  // Create local MC context to isolate the effect of ephemeral code emission.2584  IndependentCodeEmitter MCEInstance = createIndependentMCCodeEmitter();2585  MCContext *LocalCtx = MCEInstance.LocalCtx.get();2586  MCAsmBackend *MAB =2587      TheTarget->createMCAsmBackend(*STI, *MRI, MCTargetOptions());2588 2589  SmallString<256> Code;2590  raw_svector_ostream VecOS(Code);2591 2592  std::unique_ptr<MCObjectWriter> OW = MAB->createObjectWriter(VecOS);2593  std::unique_ptr<MCStreamer> Streamer(TheTarget->createMCObjectStreamer(2594      *TheTriple, *LocalCtx, std::unique_ptr<MCAsmBackend>(MAB), std::move(OW),2595      std::unique_ptr<MCCodeEmitter>(MCEInstance.MCE.release()), *STI));2596 2597  Streamer->initSections(false, *STI);2598 2599  MCSection *Section = MCEInstance.LocalMOFI->getTextSection();2600  Section->setHasInstructions(true);2601 2602  // Create symbols in the LocalCtx so that they get destroyed with it.2603  MCSymbol *StartLabel = LocalCtx->createTempSymbol();2604  MCSymbol *EndLabel = LocalCtx->createTempSymbol();2605 2606  Streamer->switchSection(Section);2607  Streamer->emitLabel(StartLabel);2608  emitFunctionBody(*Streamer, BF, BF.getLayout().getMainFragment(),2609                   /*EmitCodeOnly=*/true);2610  Streamer->emitLabel(EndLabel);2611 2612  using LabelRange = std::pair<const MCSymbol *, const MCSymbol *>;2613  SmallVector<LabelRange> SplitLabels;2614  for (FunctionFragment &FF : BF.getLayout().getSplitFragments()) {2615    MCSymbol *const SplitStartLabel = LocalCtx->createTempSymbol();2616    MCSymbol *const SplitEndLabel = LocalCtx->createTempSymbol();2617    SplitLabels.emplace_back(SplitStartLabel, SplitEndLabel);2618 2619    MCSectionELF *const SplitSection = LocalCtx->getELFSection(2620        BF.getCodeSectionName(FF.getFragmentNum()), ELF::SHT_PROGBITS,2621        ELF::SHF_EXECINSTR | ELF::SHF_ALLOC);2622    SplitSection->setHasInstructions(true);2623    Streamer->switchSection(SplitSection);2624 2625    Streamer->emitLabel(SplitStartLabel);2626    emitFunctionBody(*Streamer, BF, FF, /*EmitCodeOnly=*/true);2627    Streamer->emitLabel(SplitEndLabel);2628  }2629 2630  MCAssembler &Assembler =2631      static_cast<MCObjectStreamer *>(Streamer.get())->getAssembler();2632  Assembler.layout();2633 2634  // Obtain fragment sizes.2635  std::vector<uint64_t> FragmentSizes;2636  // Main fragment size.2637  const uint64_t HotSize = Assembler.getSymbolOffset(*EndLabel) -2638                           Assembler.getSymbolOffset(*StartLabel);2639  FragmentSizes.push_back(HotSize);2640  // Split fragment sizes.2641  uint64_t ColdSize = 0;2642  for (const auto &Labels : SplitLabels) {2643    uint64_t Size = Assembler.getSymbolOffset(*Labels.second) -2644                    Assembler.getSymbolOffset(*Labels.first);2645    FragmentSizes.push_back(Size);2646    ColdSize += Size;2647  }2648 2649  // Populate new start and end offsets of each basic block.2650  uint64_t FragmentIndex = 0;2651  for (FunctionFragment &FF : BF.getLayout().fragments()) {2652    BinaryBasicBlock *PrevBB = nullptr;2653    for (BinaryBasicBlock *BB : FF) {2654      const uint64_t BBStartOffset =2655          Assembler.getSymbolOffset(*(BB->getLabel()));2656      BB->setOutputStartAddress(BBStartOffset);2657      if (PrevBB)2658        PrevBB->setOutputEndAddress(BBStartOffset);2659      PrevBB = BB;2660    }2661    if (PrevBB)2662      PrevBB->setOutputEndAddress(FragmentSizes[FragmentIndex]);2663    FragmentIndex++;2664  }2665 2666  // Clean-up the effect of the code emission.2667  for (const MCSymbol &Symbol : Assembler.symbols()) {2668    MCSymbol *MutableSymbol = const_cast<MCSymbol *>(&Symbol);2669    MutableSymbol->setFragment(nullptr);2670    MutableSymbol->setIsRegistered(false);2671  }2672 2673  return std::make_pair(HotSize, ColdSize);2674}2675 2676bool BinaryContext::validateInstructionEncoding(2677    ArrayRef<uint8_t> InputSequence) const {2678  MCInst Inst;2679  uint64_t InstSize;2680  DisAsm->getInstruction(Inst, InstSize, InputSequence, 0, nulls());2681  assert(InstSize == InputSequence.size() &&2682         "Disassembled instruction size does not match the sequence.");2683 2684  SmallString<256> Code;2685  SmallVector<MCFixup, 4> Fixups;2686 2687  MCE->encodeInstruction(Inst, Code, Fixups, *STI);2688  auto OutputSequence = ArrayRef<uint8_t>((uint8_t *)Code.data(), Code.size());2689  if (InputSequence != OutputSequence) {2690    if (opts::Verbosity > 1) {2691      this->errs() << "BOLT-WARNING: mismatched encoding detected\n"2692                   << "      input: " << InputSequence << '\n'2693                   << "     output: " << OutputSequence << '\n';2694    }2695    return false;2696  }2697 2698  return true;2699}2700 2701uint64_t BinaryContext::getHotThreshold() const {2702  static uint64_t Threshold = 0;2703  if (Threshold == 0) {2704    Threshold = std::max(2705        (uint64_t)opts::ExecutionCountThreshold,2706        NumProfiledFuncs ? SumExecutionCount / (2 * NumProfiledFuncs) : 1);2707  }2708  return Threshold;2709}2710 2711BinaryFunction *BinaryContext::getBinaryFunctionContainingAddress(2712    uint64_t Address, bool CheckPastEnd, bool UseMaxSize) {2713  auto FI = BinaryFunctions.upper_bound(Address);2714  if (FI == BinaryFunctions.begin())2715    return nullptr;2716  --FI;2717 2718  const uint64_t UsedSize =2719      UseMaxSize ? FI->second.getMaxSize() : FI->second.getSize();2720 2721  if (Address >= FI->first + UsedSize + (CheckPastEnd ? 1 : 0))2722    return nullptr;2723 2724  return &FI->second;2725}2726 2727BinaryFunction *BinaryContext::getBinaryFunctionAtAddress(uint64_t Address) {2728  // First, try to find a function starting at the given address. If the2729  // function was folded, this will get us the original folded function if it2730  // wasn't removed from the list, e.g. in non-relocation mode.2731  auto BFI = BinaryFunctions.find(Address);2732  if (BFI != BinaryFunctions.end())2733    return &BFI->second;2734 2735  // We might have folded the function matching the object at the given2736  // address. In such case, we look for a function matching the symbol2737  // registered at the original address. The new function (the one that the2738  // original was folded into) will hold the symbol.2739  if (const BinaryData *BD = getBinaryDataAtAddress(Address)) {2740    uint64_t EntryID = 0;2741    BinaryFunction *BF = getFunctionForSymbol(BD->getSymbol(), &EntryID);2742    if (BF && EntryID == 0)2743      return BF;2744  }2745  return nullptr;2746}2747 2748/// Deregister JumpTable registered at a given \p Address and delete it.2749void BinaryContext::deleteJumpTable(uint64_t Address) {2750  assert(JumpTables.count(Address) && "Must have a jump table at address");2751  JumpTable *JT = JumpTables.at(Address);2752  for (BinaryFunction *Parent : JT->Parents)2753    Parent->JumpTables.erase(Address);2754  JumpTables.erase(Address);2755  delete JT;2756}2757 2758DebugAddressRangesVector BinaryContext::translateModuleAddressRanges(2759    const DWARFAddressRangesVector &InputRanges) const {2760  DebugAddressRangesVector OutputRanges;2761 2762  for (const DWARFAddressRange Range : InputRanges) {2763    auto BFI = BinaryFunctions.lower_bound(Range.LowPC);2764    while (BFI != BinaryFunctions.end()) {2765      const BinaryFunction &Function = BFI->second;2766      if (Function.getAddress() >= Range.HighPC)2767        break;2768      const DebugAddressRangesVector FunctionRanges =2769          Function.getOutputAddressRanges();2770      llvm::move(FunctionRanges, std::back_inserter(OutputRanges));2771      std::advance(BFI, 1);2772    }2773  }2774 2775  return OutputRanges;2776}2777 2778} // namespace bolt2779} // namespace llvm2780