brintos / llvm-project-archived public Read only

1//===- bolt/Rewrite/RewriteInstance.cpp - ELF rewriter --------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "bolt/Rewrite/RewriteInstance.h"10#include "bolt/Core/AddressMap.h"11#include "bolt/Core/BinaryContext.h"12#include "bolt/Core/BinaryEmitter.h"13#include "bolt/Core/BinaryFunction.h"14#include "bolt/Core/DebugData.h"15#include "bolt/Core/Exceptions.h"16#include "bolt/Core/FunctionLayout.h"17#include "bolt/Core/MCPlusBuilder.h"18#include "bolt/Core/ParallelUtilities.h"19#include "bolt/Core/Relocation.h"20#include "bolt/Passes/BinaryPasses.h"21#include "bolt/Passes/CacheMetrics.h"22#include "bolt/Passes/IdenticalCodeFolding.h"23#include "bolt/Passes/PAuthGadgetScanner.h"24#include "bolt/Passes/ReorderFunctions.h"25#include "bolt/Profile/BoltAddressTranslation.h"26#include "bolt/Profile/DataAggregator.h"27#include "bolt/Profile/DataReader.h"28#include "bolt/Profile/YAMLProfileReader.h"29#include "bolt/Profile/YAMLProfileWriter.h"30#include "bolt/Rewrite/BinaryPassManager.h"31#include "bolt/Rewrite/DWARFRewriter.h"32#include "bolt/Rewrite/ExecutableFileMemoryManager.h"33#include "bolt/Rewrite/JITLinkLinker.h"34#include "bolt/Rewrite/MetadataRewriters.h"35#include "bolt/RuntimeLibs/HugifyRuntimeLibrary.h"36#include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"37#include "bolt/Utils/CommandLineOpts.h"38#include "bolt/Utils/Utils.h"39#include "llvm/ADT/AddressRanges.h"40#include "llvm/ADT/STLExtras.h"41#include "llvm/DebugInfo/DWARF/DWARFContext.h"42#include "llvm/DebugInfo/DWARF/DWARFDebugFrame.h"43#include "llvm/MC/MCAsmBackend.h"44#include "llvm/MC/MCAsmInfo.h"45#include "llvm/MC/MCDisassembler/MCDisassembler.h"46#include "llvm/MC/MCObjectStreamer.h"47#include "llvm/MC/MCStreamer.h"48#include "llvm/MC/MCSymbol.h"49#include "llvm/MC/TargetRegistry.h"50#include "llvm/Object/ObjectFile.h"51#include "llvm/Support/Alignment.h"52#include "llvm/Support/Casting.h"53#include "llvm/Support/CommandLine.h"54#include "llvm/Support/DataExtractor.h"55#include "llvm/Support/Errc.h"56#include "llvm/Support/Error.h"57#include "llvm/Support/FileSystem.h"58#include "llvm/Support/ManagedStatic.h"59#include "llvm/Support/Timer.h"60#include "llvm/Support/ToolOutputFile.h"61#include "llvm/Support/raw_ostream.h"62#include <algorithm>63#include <fstream>64#include <memory>65#include <optional>66#include <system_error>67 68#undef  DEBUG_TYPE69#define DEBUG_TYPE "bolt"70 71using namespace llvm;72using namespace object;73using namespace bolt;74 75extern cl::opt<uint32_t> X86AlignBranchBoundary;76extern cl::opt<bool> X86AlignBranchWithin32BBoundaries;77 78namespace opts {79 80extern cl::list<std::string> HotTextMoveSections;81extern cl::opt<bool> Hugify;82extern cl::opt<bool> Instrument;83extern cl::opt<bool> KeepNops;84extern cl::opt<bool> Lite;85extern cl::list<std::string> PrintOnly;86extern cl::opt<std::string> PrintOnlyFile;87extern cl::list<std::string> ReorderData;88extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;89extern cl::opt<bool> TerminalHLT;90extern cl::opt<bool> TerminalTrap;91extern cl::opt<bool> TimeBuild;92extern cl::opt<bool> TimeRewrite;93extern cl::opt<bolt::IdenticalCodeFolding::ICFLevel, false,94               llvm::bolt::DeprecatedICFNumericOptionParser>95    ICF;96 97static cl::opt<bool>98    AllowStripped("allow-stripped",99                  cl::desc("allow processing of stripped binaries"), cl::Hidden,100                  cl::cat(BoltCategory));101 102static cl::opt<bool> ForceToDataRelocations(103    "force-data-relocations",104    cl::desc("force relocations to data sections to always be processed"),105 106    cl::Hidden, cl::cat(BoltCategory));107 108static cl::opt<std::string>109    BoltID("bolt-id",110           cl::desc("add any string to tag this execution in the "111                    "output binary via bolt info section"),112           cl::cat(BoltCategory));113 114cl::opt<bool> DumpDotAll(115    "dump-dot-all",116    cl::desc("dump function CFGs to graphviz format after each stage;"117             "enable '-print-loops' for color-coded blocks"),118    cl::Hidden, cl::cat(BoltCategory));119 120cl::list<std::string> DumpDotFunc(121    "dump-dot-func", cl::CommaSeparated,122    cl::desc(123        "dump function CFGs to graphviz format for specified functions only;"124        "takes function name patterns (regex supported)"),125    cl::value_desc("func1,func2,func3,..."), cl::Hidden, cl::cat(BoltCategory));126 127bool shouldDumpDot(const bolt::BinaryFunction &Function) {128  // If dump-dot-all is enabled, dump all functions129  if (DumpDotAll)130    return !Function.isIgnored();131 132  // If no specific functions specified in dump-dot-func, don't dump any133  if (DumpDotFunc.empty())134    return false;135 136  if (Function.isIgnored())137    return false;138 139  // Check if function matches any of the specified patterns140  for (const std::string &Name : DumpDotFunc) {141    if (Function.hasNameRegex(Name)) {142      return true;143    }144  }145 146  return false;147}148 149static cl::list<std::string>150ForceFunctionNames("funcs",151  cl::CommaSeparated,152  cl::desc("limit optimizations to functions from the list"),153  cl::value_desc("func1,func2,func3,..."),154  cl::Hidden,155  cl::cat(BoltCategory));156 157static cl::opt<std::string>158FunctionNamesFile("funcs-file",159  cl::desc("file with list of functions to optimize"),160  cl::Hidden,161  cl::cat(BoltCategory));162 163static cl::list<std::string> ForceFunctionNamesNR(164    "funcs-no-regex", cl::CommaSeparated,165    cl::desc("limit optimizations to functions from the list (non-regex)"),166    cl::value_desc("func1,func2,func3,..."), cl::Hidden, cl::cat(BoltCategory));167 168static cl::opt<std::string> FunctionNamesFileNR(169    "funcs-file-no-regex",170    cl::desc("file with list of functions to optimize (non-regex)"), cl::Hidden,171    cl::cat(BoltCategory));172 173cl::opt<bool>174KeepTmp("keep-tmp",175  cl::desc("preserve intermediate .o file"),176  cl::Hidden,177  cl::cat(BoltCategory));178 179static cl::opt<unsigned>180LiteThresholdPct("lite-threshold-pct",181  cl::desc("threshold (in percent) for selecting functions to process in lite "182            "mode. Higher threshold means fewer functions to process. E.g "183            "threshold of 90 means only top 10 percent of functions with "184            "profile will be processed."),185  cl::init(0),186  cl::ZeroOrMore,187  cl::Hidden,188  cl::cat(BoltOptCategory));189 190static cl::opt<unsigned> LiteThresholdCount(191    "lite-threshold-count",192    cl::desc("similar to '-lite-threshold-pct' but specify threshold using "193             "absolute function call count. I.e. limit processing to functions "194             "executed at least the specified number of times."),195    cl::init(0), cl::Hidden, cl::cat(BoltOptCategory));196 197static cl::opt<unsigned>198    MaxFunctions("max-funcs",199                 cl::desc("maximum number of functions to process"), cl::Hidden,200                 cl::cat(BoltCategory));201 202static cl::opt<unsigned> MaxDataRelocations(203    "max-data-relocations",204    cl::desc("maximum number of data relocations to process"), cl::Hidden,205    cl::cat(BoltCategory));206 207cl::opt<bool> PrintAll("print-all",208                       cl::desc("print functions after each stage"), cl::Hidden,209                       cl::cat(BoltCategory));210 211static cl::opt<bool>212    PrintProfile("print-profile",213                 cl::desc("print functions after attaching profile"),214                 cl::Hidden, cl::cat(BoltCategory));215 216cl::opt<bool> PrintCFG("print-cfg",217                       cl::desc("print functions after CFG construction"),218                       cl::Hidden, cl::cat(BoltCategory));219 220cl::opt<bool> PrintDisasm("print-disasm",221                          cl::desc("print function after disassembly"),222                          cl::Hidden, cl::cat(BoltCategory));223 224static cl::opt<bool>225    PrintGlobals("print-globals",226                 cl::desc("print global symbols after disassembly"), cl::Hidden,227                 cl::cat(BoltCategory));228 229extern cl::opt<bool> PrintSections;230 231static cl::opt<bool> PrintLoopInfo("print-loops",232                                   cl::desc("print loop related information"),233                                   cl::Hidden, cl::cat(BoltCategory));234 235static cl::opt<cl::boolOrDefault> RelocationMode(236    "relocs", cl::desc("use relocations in the binary (default=autodetect)"),237    cl::cat(BoltCategory));238 239extern cl::opt<std::string> SaveProfile;240 241static cl::list<std::string>242SkipFunctionNames("skip-funcs",243  cl::CommaSeparated,244  cl::desc("list of functions to skip"),245  cl::value_desc("func1,func2,func3,..."),246  cl::Hidden,247  cl::cat(BoltCategory));248 249static cl::opt<std::string>250SkipFunctionNamesFile("skip-funcs-file",251  cl::desc("file with list of functions to skip"),252  cl::Hidden,253  cl::cat(BoltCategory));254 255static cl::opt<bool> TrapOldCode(256    "trap-old-code",257    cl::desc("insert traps in old function bodies (relocation mode)"),258    cl::Hidden, cl::cat(BoltCategory));259 260static cl::opt<std::string> DWPPathName("dwp",261                                        cl::desc("Path and name to DWP file."),262                                        cl::Hidden, cl::init(""),263                                        cl::cat(BoltCategory));264 265static cl::opt<bool>266UseGnuStack("use-gnu-stack",267  cl::desc("use GNU_STACK program header for new segment (workaround for "268           "issues with strip/objcopy)"),269  cl::ZeroOrMore,270  cl::cat(BoltCategory));271 272static cl::opt<uint64_t> CustomAllocationVMA(273    "custom-allocation-vma",274    cl::desc("use a custom address at which new code will be put, "275             "bypassing BOLT's logic to detect where to put code"),276    cl::Hidden, cl::cat(BoltCategory));277 278static cl::opt<bool>279SequentialDisassembly("sequential-disassembly",280  cl::desc("performs disassembly sequentially"),281  cl::init(false),282  cl::cat(BoltOptCategory));283 284static cl::opt<bool> WriteBoltInfoSection(285    "bolt-info", cl::desc("write bolt info section in the output binary"),286    cl::init(true), cl::Hidden, cl::cat(BoltOutputCategory));287 288cl::bits<GadgetScannerKind> GadgetScannersToRun(289    "scanners", cl::desc("which gadget scanners to run"),290    cl::values(291        clEnumValN(GS_PACRET, "pacret",292                   "pac-ret: return address protection (subset of \"pauth\")"),293        clEnumValN(GS_PAUTH, "pauth", "All Pointer Authentication scanners"),294        clEnumValN(GS_ALL, "all", "All implemented scanners")),295    cl::ZeroOrMore, cl::CommaSeparated, cl::cat(BinaryAnalysisCategory));296 297// Primary targets for hooking runtime library initialization hooking298// with fallback to next item in case if current item is not available299// in the input binary.300enum RuntimeLibInitHookTarget : char {301  RLIH_ENTRY_POINT = 0, /// Use ELF Header Entry Point302  RLIH_INIT = 1,        /// Use ELF DT_INIT entry303  RLIH_INIT_ARRAY = 2,  /// Use ELF .init_array entry304};305 306cl::opt<RuntimeLibInitHookTarget> RuntimeLibInitHook(307    "runtime-lib-init-hook",308    cl::desc("Primary target for hooking runtime library initialization, used "309             "in fallback order of availabiliy in input binary (entry_point -> "310             "init -> init_array) (default: entry_point)"),311    cl::Hidden, cl::init(RLIH_ENTRY_POINT),312    cl::values(clEnumValN(RLIH_ENTRY_POINT, "entry_point",313                          "use ELF Header Entry Point"),314               clEnumValN(RLIH_INIT, "init", "use ELF DT_INIT entry"),315               clEnumValN(RLIH_INIT_ARRAY, "init_array",316                          "use ELF .init_array entry")),317    cl::ZeroOrMore, cl::cat(BoltOptCategory));318 319} // namespace opts320 321// FIXME: implement a better way to mark sections for replacement.322std::vector<std::string> RewriteInstance::DebugSectionsToOverwrite = {323    ".debug_abbrev", ".debug_aranges",  ".debug_line",   ".debug_line_str",324    ".debug_loc",    ".debug_loclists", ".debug_ranges", ".debug_rnglists",325    ".gdb_index",    ".debug_addr",     ".debug_abbrev", ".debug_info",326    ".debug_types",  ".pseudo_probe"};327 328const char RewriteInstance::TimerGroupName[] = "rewrite";329const char RewriteInstance::TimerGroupDesc[] = "Rewrite passes";330 331namespace llvm {332namespace bolt {333 334extern const char *BoltRevision;335 336// Weird location for createMCPlusBuilder, but this is here to avoid a337// cyclic dependency of libCore (its natural place) and libTarget. libRewrite338// can depend on libTarget, but not libCore. Since libRewrite is the only339// user of this function, we define it here.340MCPlusBuilder *createMCPlusBuilder(const Triple::ArchType Arch,341                                   const MCInstrAnalysis *Analysis,342                                   const MCInstrInfo *Info,343                                   const MCRegisterInfo *RegInfo,344                                   const MCSubtargetInfo *STI) {345#ifdef X86_AVAILABLE346  if (Arch == Triple::x86_64)347    return createX86MCPlusBuilder(Analysis, Info, RegInfo, STI);348#endif349 350#ifdef AARCH64_AVAILABLE351  if (Arch == Triple::aarch64)352    return createAArch64MCPlusBuilder(Analysis, Info, RegInfo, STI);353#endif354 355#ifdef RISCV_AVAILABLE356  if (Arch == Triple::riscv64)357    return createRISCVMCPlusBuilder(Analysis, Info, RegInfo, STI);358#endif359 360  llvm_unreachable("architecture unsupported by MCPlusBuilder");361}362 363} // namespace bolt364} // namespace llvm365 366using ELF64LEPhdrTy = ELF64LEFile::Elf_Phdr;367 368namespace {369 370bool refersToReorderedSection(ErrorOr<BinarySection &> Section) {371  return llvm::any_of(opts::ReorderData, [&](const std::string &SectionName) {372    return Section && Section->getName() == SectionName;373  });374}375 376} // anonymous namespace377 378Expected<std::unique_ptr<RewriteInstance>>379RewriteInstance::create(ELFObjectFileBase *File, const int Argc,380                        const char *const *Argv, StringRef ToolPath,381                        raw_ostream &Stdout, raw_ostream &Stderr) {382  Error Err = Error::success();383  auto RI = std::make_unique<RewriteInstance>(File, Argc, Argv, ToolPath,384                                              Stdout, Stderr, Err);385  if (Err)386    return std::move(Err);387  return std::move(RI);388}389 390RewriteInstance::RewriteInstance(ELFObjectFileBase *File, const int Argc,391                                 const char *const *Argv, StringRef ToolPath,392                                 raw_ostream &Stdout, raw_ostream &Stderr,393                                 Error &Err)394    : InputFile(File), Argc(Argc), Argv(Argv), ToolPath(ToolPath),395      SHStrTab(StringTableBuilder::ELF) {396  ErrorAsOutParameter EAO(&Err);397  auto ELF64LEFile = dyn_cast<ELF64LEObjectFile>(InputFile);398  if (!ELF64LEFile) {399    Err = createStringError(errc::not_supported,400                            "Only 64-bit LE ELF binaries are supported");401    return;402  }403 404  bool IsPIC = false;405  const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();406  if (Obj.getHeader().e_type != ELF::ET_EXEC) {407    Stdout << "BOLT-INFO: shared object or position-independent executable "408              "detected\n";409    IsPIC = true;410  }411 412  // Make sure we don't miss any output on core dumps.413  Stdout.SetUnbuffered();414  Stderr.SetUnbuffered();415  LLVM_DEBUG(dbgs().SetUnbuffered());416 417  // Read RISCV subtarget features from input file418  std::unique_ptr<SubtargetFeatures> Features;419  Triple TheTriple = File->makeTriple();420  if (TheTriple.getArch() == llvm::Triple::riscv64) {421    Expected<SubtargetFeatures> FeaturesOrErr = File->getFeatures();422    if (auto E = FeaturesOrErr.takeError()) {423      Err = std::move(E);424      return;425    } else {426      Features.reset(new SubtargetFeatures(*FeaturesOrErr));427    }428  }429 430  Relocation::Arch = TheTriple.getArch();431  auto BCOrErr = BinaryContext::createBinaryContext(432      TheTriple, std::make_shared<orc::SymbolStringPool>(), File->getFileName(),433      Features.get(), IsPIC,434      DWARFContext::create(*File, DWARFContext::ProcessDebugRelocations::Ignore,435                           nullptr, opts::DWPPathName,436                           WithColor::defaultErrorHandler,437                           WithColor::defaultWarningHandler),438      JournalingStreams{Stdout, Stderr});439  if (Error E = BCOrErr.takeError()) {440    Err = std::move(E);441    return;442  }443  BC = std::move(BCOrErr.get());444  BC->initializeTarget(std::unique_ptr<MCPlusBuilder>(445      createMCPlusBuilder(BC->TheTriple->getArch(), BC->MIA.get(),446                          BC->MII.get(), BC->MRI.get(), BC->STI.get())));447 448  BAT = std::make_unique<BoltAddressTranslation>();449 450  if (opts::UpdateDebugSections)451    DebugInfoRewriter = std::make_unique<DWARFRewriter>(*BC);452 453  if (opts::Instrument)454    BC->setRuntimeLibrary(std::make_unique<InstrumentationRuntimeLibrary>());455  else if (opts::Hugify)456    BC->setRuntimeLibrary(std::make_unique<HugifyRuntimeLibrary>());457}458 459RewriteInstance::~RewriteInstance() {}460 461Error RewriteInstance::setProfile(StringRef Filename) {462  if (!sys::fs::exists(Filename))463    return errorCodeToError(make_error_code(errc::no_such_file_or_directory));464 465  if (ProfileReader) {466    // Already exists467    return make_error<StringError>(Twine("multiple profiles specified: ") +468                                       ProfileReader->getFilename() + " and " +469                                       Filename,470                                   inconvertibleErrorCode());471  }472 473  // Spawn a profile reader based on file contents.474  if (DataAggregator::checkPerfDataMagic(Filename))475    ProfileReader = std::make_unique<DataAggregator>(Filename);476  else if (YAMLProfileReader::isYAML(Filename))477    ProfileReader = std::make_unique<YAMLProfileReader>(Filename);478  else479    ProfileReader = std::make_unique<DataReader>(Filename);480 481  return Error::success();482}483 484/// Return true if the function \p BF should be disassembled.485static bool shouldDisassemble(const BinaryFunction &BF) {486  if (BF.isPseudo())487    return false;488 489  if (opts::processAllFunctions())490    return true;491 492  return !BF.isIgnored();493}494 495// Return if a section stored in the image falls into a segment address space.496// If not, Set \p Overlap to true if there's a partial overlap.497template <class ELFT>498static bool checkOffsets(const typename ELFT::Phdr &Phdr,499                         const typename ELFT::Shdr &Sec, bool &Overlap) {500  // SHT_NOBITS sections don't need to have an offset inside the segment.501  if (Sec.sh_type == ELF::SHT_NOBITS)502    return true;503 504  // Only non-empty sections can be at the end of a segment.505  uint64_t SectionSize = Sec.sh_size ? Sec.sh_size : 1ull;506  AddressRange SectionAddressRange((uint64_t)Sec.sh_offset,507                                   Sec.sh_offset + SectionSize);508  AddressRange SegmentAddressRange(Phdr.p_offset,509                                   Phdr.p_offset + Phdr.p_filesz);510  if (SegmentAddressRange.contains(SectionAddressRange))511    return true;512 513  Overlap = SegmentAddressRange.intersects(SectionAddressRange);514  return false;515}516 517// Check that an allocatable section belongs to a virtual address518// space of a segment.519template <class ELFT>520static bool checkVMA(const typename ELFT::Phdr &Phdr,521                     const typename ELFT::Shdr &Sec, bool &Overlap) {522  // Only non-empty sections can be at the end of a segment.523  uint64_t SectionSize = Sec.sh_size ? Sec.sh_size : 1ull;524  AddressRange SectionAddressRange((uint64_t)Sec.sh_addr,525                                   Sec.sh_addr + SectionSize);526  AddressRange SegmentAddressRange(Phdr.p_vaddr, Phdr.p_vaddr + Phdr.p_memsz);527 528  if (SegmentAddressRange.contains(SectionAddressRange))529    return true;530  Overlap = SegmentAddressRange.intersects(SectionAddressRange);531  return false;532}533 534void RewriteInstance::markGnuRelroSections() {535  using ELFT = ELF64LE;536  using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;537  auto ELF64LEFile = cast<ELF64LEObjectFile>(InputFile);538  const ELFFile<ELFT> &Obj = ELF64LEFile->getELFFile();539 540  auto handleSection = [&](const ELFT::Phdr &Phdr, SectionRef SecRef) {541    BinarySection *BinarySection = BC->getSectionForSectionRef(SecRef);542    // If the section is non-allocatable, ignore it for GNU_RELRO purposes:543    // it can't be made read-only after runtime relocations processing.544    if (!BinarySection || !BinarySection->isAllocatable())545      return;546    const ELFShdrTy *Sec = cantFail(Obj.getSection(SecRef.getIndex()));547    bool ImageOverlap{false}, VMAOverlap{false};548    bool ImageContains = checkOffsets<ELFT>(Phdr, *Sec, ImageOverlap);549    bool VMAContains = checkVMA<ELFT>(Phdr, *Sec, VMAOverlap);550    if (ImageOverlap) {551      if (opts::Verbosity >= 1)552        BC->errs() << "BOLT-WARNING: GNU_RELRO segment has partial file offset "553                   << "overlap with section " << BinarySection->getName()554                   << '\n';555      return;556    }557    if (VMAOverlap) {558      if (opts::Verbosity >= 1)559        BC->errs() << "BOLT-WARNING: GNU_RELRO segment has partial VMA overlap "560                   << "with section " << BinarySection->getName() << '\n';561      return;562    }563    if (!ImageContains || !VMAContains)564      return;565    BinarySection->setRelro();566    if (opts::Verbosity >= 1)567      BC->outs() << "BOLT-INFO: marking " << BinarySection->getName()568                 << " as GNU_RELRO\n";569  };570 571  for (const ELFT::Phdr &Phdr : cantFail(Obj.program_headers()))572    if (Phdr.p_type == ELF::PT_GNU_RELRO)573      for (SectionRef SecRef : InputFile->sections())574        handleSection(Phdr, SecRef);575}576 577Error RewriteInstance::discoverStorage() {578  NamedRegionTimer T("discoverStorage", "discover storage", TimerGroupName,579                     TimerGroupDesc, opts::TimeRewrite);580 581  auto ELF64LEFile = cast<ELF64LEObjectFile>(InputFile);582  const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();583 584  BC->StartFunctionAddress = Obj.getHeader().e_entry;585 586  NextAvailableAddress = 0;587  uint64_t NextAvailableOffset = 0;588  Expected<ELF64LE::PhdrRange> PHsOrErr = Obj.program_headers();589  if (Error E = PHsOrErr.takeError())590    return E;591 592  ELF64LE::PhdrRange PHs = PHsOrErr.get();593  for (const ELF64LE::Phdr &Phdr : PHs) {594    switch (Phdr.p_type) {595    case ELF::PT_LOAD:596      BC->FirstAllocAddress = std::min(BC->FirstAllocAddress,597                                       static_cast<uint64_t>(Phdr.p_vaddr));598      NextAvailableAddress = std::max(NextAvailableAddress,599                                      Phdr.p_vaddr + Phdr.p_memsz);600      NextAvailableOffset = std::max(NextAvailableOffset,601                                     Phdr.p_offset + Phdr.p_filesz);602 603      BC->SegmentMapInfo[Phdr.p_vaddr] =604          SegmentInfo{Phdr.p_vaddr,605                      Phdr.p_memsz,606                      Phdr.p_offset,607                      Phdr.p_filesz,608                      Phdr.p_align,609                      (Phdr.p_flags & ELF::PF_X) != 0,610                      (Phdr.p_flags & ELF::PF_W) != 0};611      if (BC->TheTriple->getArch() == llvm::Triple::x86_64 &&612          Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)613        BC->IsLinuxKernel = true;614      break;615    case ELF::PT_INTERP:616      BC->HasInterpHeader = true;617      break;618    }619  }620 621  if (BC->IsLinuxKernel)622    BC->outs() << "BOLT-INFO: Linux kernel binary detected\n";623 624  for (const SectionRef &Section : InputFile->sections()) {625    Expected<StringRef> SectionNameOrErr = Section.getName();626    if (Error E = SectionNameOrErr.takeError())627      return E;628    StringRef SectionName = SectionNameOrErr.get();629    if (SectionName == BC->getMainCodeSectionName()) {630      BC->OldTextSectionAddress = Section.getAddress();631      BC->OldTextSectionSize = Section.getSize();632 633      Expected<StringRef> SectionContentsOrErr = Section.getContents();634      if (Error E = SectionContentsOrErr.takeError())635        return E;636      StringRef SectionContents = SectionContentsOrErr.get();637      BC->OldTextSectionOffset =638          SectionContents.data() - InputFile->getData().data();639    }640 641    if (!opts::HeatmapMode &&642        !(opts::AggregateOnly && BAT->enabledFor(InputFile)) &&643        (SectionName.starts_with(getOrgSecPrefix()) ||644         SectionName == getBOLTTextSectionName()))645      return createStringError(646          errc::function_not_supported,647          "BOLT-ERROR: input file was processed by BOLT. Cannot re-optimize");648  }649 650  if (!NextAvailableAddress || !NextAvailableOffset)651    return createStringError(errc::executable_format_error,652                             "no PT_LOAD pheader seen");653 654  BC->outs() << "BOLT-INFO: first alloc address is 0x"655             << Twine::utohexstr(BC->FirstAllocAddress) << '\n';656 657  FirstNonAllocatableOffset = NextAvailableOffset;658 659  if (opts::CustomAllocationVMA) {660    // If user specified a custom address where we should start writing new661    // data, honor that.662    NextAvailableAddress = opts::CustomAllocationVMA;663    // Sanity check the user-supplied address and emit warnings if something664    // seems off.665    for (const ELF64LE::Phdr &Phdr : PHs) {666      switch (Phdr.p_type) {667      case ELF::PT_LOAD:668        if (NextAvailableAddress >= Phdr.p_vaddr &&669            NextAvailableAddress < Phdr.p_vaddr + Phdr.p_memsz) {670          BC->errs() << "BOLT-WARNING: user-supplied allocation vma 0x"671                     << Twine::utohexstr(NextAvailableAddress)672                     << " conflicts with ELF segment at 0x"673                     << Twine::utohexstr(Phdr.p_vaddr) << "\n";674        }675      }676    }677  }678  NextAvailableAddress = alignTo(NextAvailableAddress, BC->PageAlign);679  NextAvailableOffset = alignTo(NextAvailableOffset, BC->PageAlign);680 681  // Hugify: Additional huge page from left side due to682  // weird ASLR mapping addresses (4KB aligned)683  if (opts::Hugify && !BC->HasFixedLoadAddress) {684    NextAvailableAddress += BC->PageAlign;685  }686 687  NewTextSegmentAddress = NextAvailableAddress;688  NewTextSegmentOffset = NextAvailableOffset;689 690  if (!opts::UseGnuStack && !BC->IsLinuxKernel) {691    // This is where the black magic happens. Creating PHDR table in a segment692    // other than that containing ELF header is tricky. Some loaders and/or693    // parts of loaders will apply e_phoff from ELF header assuming both are in694    // the same segment, while others will do the proper calculation.695    // We create the new PHDR table in such a way that both of the methods696    // of loading and locating the table work. There's a slight file size697    // overhead because of that.698    //699    // NB: bfd's strip command cannot do the above and will corrupt the700    //     binary during the process of stripping non-allocatable sections.701    if (NextAvailableOffset <= NextAvailableAddress - BC->FirstAllocAddress)702      NextAvailableOffset = NextAvailableAddress - BC->FirstAllocAddress;703    else704      NextAvailableAddress = NextAvailableOffset + BC->FirstAllocAddress;705 706    assert(NextAvailableOffset ==707               NextAvailableAddress - BC->FirstAllocAddress &&708           "PHDR table address calculation error");709 710    BC->outs() << "BOLT-INFO: creating new program header table at address 0x"711               << Twine::utohexstr(NextAvailableAddress) << ", offset 0x"712               << Twine::utohexstr(NextAvailableOffset) << '\n';713 714    PHDRTableAddress = NextAvailableAddress;715    PHDRTableOffset = NextAvailableOffset;716    NewTextSegmentAddress = NextAvailableAddress;717    NewTextSegmentOffset = NextAvailableOffset;718 719    // Reserve space for 3 extra pheaders.720    unsigned Phnum = Obj.getHeader().e_phnum;721    Phnum += 3;722 723    // Reserve two more pheaders to avoid having writeable and executable724    // segment in instrumented binary.725    if (opts::Instrument)726      Phnum += 2;727 728    NextAvailableAddress += Phnum * sizeof(ELF64LEPhdrTy);729    NextAvailableOffset += Phnum * sizeof(ELF64LEPhdrTy);730 731    // Align at cache line.732    NextAvailableAddress = alignTo(NextAvailableAddress, 64);733    NextAvailableOffset = alignTo(NextAvailableOffset, 64);734  }735 736  BC->LayoutStartAddress = NextAvailableAddress;737 738  // Tools such as objcopy can strip section contents but leave header739  // entries. Check that at least .text is mapped in the file.740  if (!getFileOffsetForAddress(BC->OldTextSectionAddress))741    return createStringError(errc::executable_format_error,742                             "BOLT-ERROR: input binary is not a valid ELF "743                             "executable as its text section is not "744                             "mapped to a valid segment");745  return Error::success();746}747 748Error RewriteInstance::run() {749  assert(BC && "failed to create a binary context");750 751  BC->outs() << "BOLT-INFO: Target architecture: "752             << Triple::getArchTypeName(753                    (llvm::Triple::ArchType)InputFile->getArch())754             << "\n";755  BC->outs() << "BOLT-INFO: BOLT version: " << BoltRevision << "\n";756 757  selectFunctionsToPrint();758 759  if (Error E = discoverStorage())760    return E;761  if (Error E = readSpecialSections())762    return E;763  adjustCommandLineOptions();764  discoverFileObjects();765 766  if (opts::Instrument && !BC->IsStaticExecutable) {767    if (Error E = discoverRtInitAddress())768      return E;769    if (Error E = discoverRtFiniAddress())770      return E;771  }772 773  preprocessProfileData();774 775  selectFunctionsToProcess();776 777  readDebugInfo();778 779  disassembleFunctions();780 781  processMetadataPreCFG();782 783  buildFunctionsCFG();784 785  processProfileData();786 787  // Save input binary metadata if BAT section needs to be emitted788  if (opts::EnableBAT)789    BAT->saveMetadata(*BC);790 791  postProcessFunctions();792 793  processMetadataPostCFG();794 795  if (opts::DiffOnly)796    return Error::success();797 798  if (opts::BinaryAnalysisMode) {799    runBinaryAnalyses();800    return Error::success();801  }802 803  preregisterSections();804 805  runOptimizationPasses();806 807  finalizeMetadataPreEmit();808 809  emitAndLink();810 811  updateMetadata();812 813  if (opts::Instrument && !BC->IsStaticExecutable) {814    if (Error E = updateRtInitReloc())815      return E;816    if (Error E = updateRtFiniReloc())817      return E;818  }819 820  if (opts::OutputFilename == "/dev/null") {821    BC->outs() << "BOLT-INFO: skipping writing final binary to disk\n";822    return Error::success();823  } else if (BC->IsLinuxKernel) {824    BC->errs() << "BOLT-WARNING: Linux kernel support is experimental\n";825  }826 827  // Rewrite allocatable contents and copy non-allocatable parts with mods.828  rewriteFile();829  return Error::success();830}831 832void RewriteInstance::discoverFileObjects() {833  NamedRegionTimer T("discoverFileObjects", "discover file objects",834                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);835 836  // For local symbols we want to keep track of associated FILE symbol name for837  // disambiguation by combined name.838  for (const ELFSymbolRef &Symbol : InputFile->symbols()) {839    Expected<StringRef> NameOrError = Symbol.getName();840    if (NameOrError && NameOrError->starts_with("__asan_init")) {841      BC->errs()842          << "BOLT-ERROR: input file was compiled or linked with sanitizer "843             "support. Cannot optimize.\n";844      exit(1);845    }846    if (NameOrError && NameOrError->starts_with("__llvm_coverage_mapping")) {847      BC->errs()848          << "BOLT-ERROR: input file was compiled or linked with coverage "849             "support. Cannot optimize.\n";850      exit(1);851    }852 853    if (cantFail(Symbol.getFlags()) & SymbolRef::SF_Undefined)854      continue;855 856    if (cantFail(Symbol.getType()) == SymbolRef::ST_File)857      FileSymbols.emplace_back(Symbol);858  }859 860  // Sort symbols in the file by value. Ignore symbols from non-allocatable861  // sections. We memoize getAddress(), as it has rather high overhead.862  struct SymbolInfo {863    uint64_t Address;864    SymbolRef Symbol;865  };866  std::vector<SymbolInfo> SortedSymbols;867  auto isSymbolInMemory = [this](const SymbolRef &Sym) {868    if (cantFail(Sym.getType()) == SymbolRef::ST_File)869      return false;870    if (cantFail(Sym.getFlags()) & SymbolRef::SF_Absolute)871      return true;872    if (cantFail(Sym.getFlags()) & SymbolRef::SF_Undefined)873      return false;874    BinarySection Section(*BC, *cantFail(Sym.getSection()));875    return Section.isAllocatable();876  };877  auto checkSymbolInSection = [this](const SymbolInfo &S) {878    // Sometimes, we encounter symbols with addresses outside their section. If879    // such symbols happen to fall into another section, they can interfere with880    // disassembly. Notably, this occurs with AArch64 marker symbols ($d and $t)881    // that belong to .eh_frame, but end up pointing into .text.882    // As a workaround, we ignore all symbols that lie outside their sections.883    auto Section = cantFail(S.Symbol.getSection());884 885    // Accept all absolute symbols.886    if (Section == InputFile->section_end())887      return true;888 889    uint64_t SecStart = Section->getAddress();890    uint64_t SecEnd = SecStart + Section->getSize();891    uint64_t SymEnd = S.Address + ELFSymbolRef(S.Symbol).getSize();892    if (S.Address >= SecStart && SymEnd <= SecEnd)893      return true;894 895    auto SymType = cantFail(S.Symbol.getType());896    // Skip warnings for common benign cases.897    if (opts::Verbosity < 1 && SymType == SymbolRef::ST_Other)898      return false; // E.g. ELF::STT_TLS.899 900    auto SymName = S.Symbol.getName();901    auto SecName = cantFail(S.Symbol.getSection())->getName();902    BC->errs() << "BOLT-WARNING: ignoring symbol "903               << (SymName ? *SymName : "[unnamed]") << " at 0x"904               << Twine::utohexstr(S.Address) << ", which lies outside "905               << (SecName ? *SecName : "[unnamed]") << "\n";906 907    return false;908  };909  for (const SymbolRef &Symbol : InputFile->symbols())910    if (isSymbolInMemory(Symbol)) {911      SymbolInfo SymInfo{cantFail(Symbol.getAddress()), Symbol};912      if (checkSymbolInSection(SymInfo))913        SortedSymbols.push_back(SymInfo);914    }915 916  auto CompareSymbols = [this](const SymbolInfo &A, const SymbolInfo &B) {917    if (A.Address != B.Address)918      return A.Address < B.Address;919 920    const bool AMarker = BC->isMarker(A.Symbol);921    const bool BMarker = BC->isMarker(B.Symbol);922    if (AMarker || BMarker) {923      return AMarker && !BMarker;924    }925 926    const auto AType = cantFail(A.Symbol.getType());927    const auto BType = cantFail(B.Symbol.getType());928    if (AType == SymbolRef::ST_Function && BType != SymbolRef::ST_Function)929      return true;930    if (BType == SymbolRef::ST_Debug && AType != SymbolRef::ST_Debug)931      return true;932 933    return false;934  };935  llvm::stable_sort(SortedSymbols, CompareSymbols);936 937  auto LastSymbol = SortedSymbols.end();938  if (!SortedSymbols.empty())939    --LastSymbol;940 941  // For aarch64, the ABI defines mapping symbols so we identify data in the942  // code section (see IHI0056B). $d identifies data contents.943  // Compilers usually merge multiple data objects in a single $d-$x interval,944  // but we need every data object to be marked with $d. Because of that we945  // keep track of marker symbols with all locations of data objects.946 947  DenseMap<uint64_t, MarkerSymType> MarkerSymbols;948  auto addExtraDataMarkerPerSymbol = [&]() {949    bool IsData = false;950    uint64_t LastAddr = 0;951    for (const auto &SymInfo : SortedSymbols) {952      MarkerSymType MarkerType = BC->getMarkerType(SymInfo.Symbol);953 954      // Treat ST_Function as code.955      Expected<object::SymbolRef::Type> TypeOrError = SymInfo.Symbol.getType();956      consumeError(TypeOrError.takeError());957      if (TypeOrError && *TypeOrError == SymbolRef::ST_Function) {958        if (IsData) {959          Expected<StringRef> NameOrError = SymInfo.Symbol.getName();960          consumeError(NameOrError.takeError());961          if (LastAddr == SymInfo.Address) {962            BC->errs() << "BOLT-WARNING: ignoring data marker conflicting with "963                          "function symbol "964                       << *NameOrError << '\n';965          } else {966            BC->errs() << "BOLT-WARNING: function symbol " << *NameOrError967                       << " lacks code marker\n";968          }969        }970        MarkerType = MarkerSymType::CODE;971      }972 973      if (MarkerType != MarkerSymType::NONE) {974        MarkerSymbols[SymInfo.Address] = MarkerType;975        LastAddr = SymInfo.Address;976        IsData = MarkerType == MarkerSymType::DATA;977        continue;978      }979 980      if (IsData) {981        MarkerSymbols[SymInfo.Address] = MarkerSymType::DATA;982        LastAddr = SymInfo.Address;983      }984    }985  };986 987  if (BC->isAArch64() || BC->isRISCV()) {988    addExtraDataMarkerPerSymbol();989    LastSymbol = std::stable_partition(990        SortedSymbols.begin(), SortedSymbols.end(),991        [this](const SymbolInfo &S) { return !BC->isMarker(S.Symbol); });992    if (!SortedSymbols.empty())993      --LastSymbol;994  }995 996  BinaryFunction *PreviousFunction = nullptr;997  unsigned AnonymousId = 0;998 999  const auto SortedSymbolsEnd =1000      LastSymbol == SortedSymbols.end() ? LastSymbol : std::next(LastSymbol);1001  for (auto Iter = SortedSymbols.begin(); Iter != SortedSymbolsEnd; ++Iter) {1002    const SymbolRef &Symbol = Iter->Symbol;1003    const uint64_t SymbolAddress = Iter->Address;1004    const auto SymbolFlags = cantFail(Symbol.getFlags());1005    const SymbolRef::Type SymbolType = cantFail(Symbol.getType());1006 1007    if (SymbolType == SymbolRef::ST_File)1008      continue;1009 1010    StringRef SymName = cantFail(Symbol.getName(), "cannot get symbol name");1011    if (SymbolAddress == 0) {1012      if (opts::Verbosity >= 1 && SymbolType == SymbolRef::ST_Function)1013        BC->errs() << "BOLT-WARNING: function with 0 address seen\n";1014      continue;1015    }1016 1017    // Ignore input hot markers unless in heatmap mode1018    if ((SymName == "__hot_start" || SymName == "__hot_end") &&1019        !opts::HeatmapMode)1020      continue;1021 1022    FileSymRefs.emplace(SymbolAddress, Symbol);1023 1024    // Skip section symbols that will be registered by disassemblePLT().1025    if (SymbolType == SymbolRef::ST_Debug) {1026      ErrorOr<BinarySection &> BSection =1027          BC->getSectionForAddress(SymbolAddress);1028      if (BSection && getPLTSectionInfo(BSection->getName()))1029        continue;1030    }1031 1032    /// It is possible we are seeing a globalized local. LLVM might treat it as1033    /// a local if it has a "private global" prefix, e.g. ".L". Thus we have to1034    /// change the prefix to enforce global scope of the symbol.1035    std::string Name =1036        SymName.starts_with(BC->AsmInfo->getPrivateGlobalPrefix())1037            ? "PG" + std::string(SymName)1038            : std::string(SymName);1039 1040    // Disambiguate all local symbols before adding to symbol table.1041    // Since we don't know if we will see a global with the same name,1042    // always modify the local name.1043    //1044    // NOTE: the naming convention for local symbols should match1045    //       the one we use for profile data.1046    std::string UniqueName;1047    std::string AlternativeName;1048    if (Name.empty()) {1049      UniqueName = "ANONYMOUS." + std::to_string(AnonymousId++);1050    } else if (SymbolFlags & SymbolRef::SF_Global) {1051      if (const BinaryData *BD = BC->getBinaryDataByName(Name)) {1052        if (BD->getSize() == ELFSymbolRef(Symbol).getSize() &&1053            BD->getAddress() == SymbolAddress) {1054          if (opts::Verbosity > 1)1055            BC->errs() << "BOLT-WARNING: ignoring duplicate global symbol "1056                       << Name << "\n";1057          // Ignore duplicate entry - possibly a bug in the linker1058          continue;1059        }1060        BC->errs() << "BOLT-ERROR: bad input binary, global symbol \"" << Name1061                   << "\" is not unique\n";1062        exit(1);1063      }1064      UniqueName = Name;1065    } else {1066      // If we have a local file name, we should create 2 variants for the1067      // function name. The reason is that perf profile might have been1068      // collected on a binary that did not have the local file name (e.g. as1069      // a side effect of stripping debug info from the binary):1070      //1071      //   primary:     <function>/<id>1072      //   alternative: <function>/<file>/<id2>1073      //1074      // The <id> field is used for disambiguation of local symbols since there1075      // could be identical function names coming from identical file names1076      // (e.g. from different directories).1077      auto SFI = llvm::upper_bound(FileSymbols, ELFSymbolRef(Symbol));1078      if (SymbolType == SymbolRef::ST_Function && SFI != FileSymbols.begin()) {1079        StringRef FileSymbolName = cantFail(SFI[-1].getName());1080        if (!FileSymbolName.empty())1081          AlternativeName = NR.uniquify(Name + "/" + FileSymbolName.str());1082      }1083 1084      UniqueName = NR.uniquify(Name);1085    }1086 1087    uint64_t SymbolSize = ELFSymbolRef(Symbol).getSize();1088    uint64_t SymbolAlignment = Symbol.getAlignment();1089 1090    auto registerName = [&](uint64_t FinalSize) {1091      // Register names even if it's not a function, e.g. for an entry point.1092      BC->registerNameAtAddress(UniqueName, SymbolAddress, FinalSize,1093                                SymbolAlignment, SymbolFlags);1094      if (!AlternativeName.empty())1095        BC->registerNameAtAddress(AlternativeName, SymbolAddress, FinalSize,1096                                  SymbolAlignment, SymbolFlags);1097    };1098 1099    section_iterator Section =1100        cantFail(Symbol.getSection(), "cannot get symbol section");1101    if (Section == InputFile->section_end()) {1102      // Could be an absolute symbol. Used on RISC-V for __global_pointer$ so we1103      // need to record it to handle relocations against it. For other instances1104      // of absolute symbols, we record for pretty printing.1105      LLVM_DEBUG(if (opts::Verbosity > 1) {1106        dbgs() << "BOLT-INFO: absolute sym " << UniqueName << "\n";1107      });1108      registerName(SymbolSize);1109      continue;1110    }1111 1112    if (SymName == getBOLTReservedStart() || SymName == getBOLTReservedEnd()) {1113      registerName(SymbolSize);1114      continue;1115    }1116 1117    LLVM_DEBUG(dbgs() << "BOLT-DEBUG: considering symbol " << UniqueName1118                      << " for function\n");1119 1120    if (SymbolAddress == Section->getAddress() + Section->getSize()) {1121      assert(SymbolSize == 0 &&1122             "unexpected non-zero sized symbol at end of section");1123      LLVM_DEBUG(1124          dbgs()1125          << "BOLT-DEBUG: rejecting as symbol points to end of its section\n");1126      registerName(SymbolSize);1127      continue;1128    }1129 1130    if (!Section->isText() || Section->isVirtual()) {1131      assert(SymbolType != SymbolRef::ST_Function &&1132             "unexpected function inside non-code section");1133      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: rejecting as symbol is not in code or "1134                           "is in nobits section\n");1135      registerName(SymbolSize);1136      continue;1137    }1138 1139    // Assembly functions could be ST_NONE with 0 size. Check that the1140    // corresponding section is a code section and they are not inside any1141    // other known function to consider them.1142    //1143    // Sometimes assembly functions are not marked as functions and neither are1144    // their local labels. The only way to tell them apart is to look at1145    // symbol scope - global vs local.1146    if (PreviousFunction && SymbolType != SymbolRef::ST_Function) {1147      if (PreviousFunction->containsAddress(SymbolAddress)) {1148        if (PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {1149          LLVM_DEBUG(dbgs()1150                     << "BOLT-DEBUG: symbol is a function local symbol\n");1151        } else if (SymbolAddress == PreviousFunction->getAddress() &&1152                   !SymbolSize) {1153          LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring symbol as a marker\n");1154        } else if (opts::Verbosity > 1) {1155          BC->errs() << "BOLT-WARNING: symbol " << UniqueName1156                     << " seen in the middle of function " << *PreviousFunction1157                     << ". Could be a new entry.\n";1158        }1159        registerName(SymbolSize);1160        continue;1161      } else if (PreviousFunction->getSize() == 0 &&1162                 PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {1163        LLVM_DEBUG(dbgs() << "BOLT-DEBUG: symbol is a function local symbol\n");1164        registerName(SymbolSize);1165        continue;1166      }1167    }1168 1169    if (PreviousFunction && PreviousFunction->containsAddress(SymbolAddress) &&1170        PreviousFunction->getAddress() != SymbolAddress) {1171      if (PreviousFunction->isSymbolValidInScope(Symbol, SymbolSize)) {1172        if (opts::Verbosity >= 1)1173          BC->outs()1174              << "BOLT-INFO: skipping possibly another entry for function "1175              << *PreviousFunction << " : " << UniqueName << '\n';1176        registerName(SymbolSize);1177      } else {1178        BC->outs() << "BOLT-INFO: using " << UniqueName1179                   << " as another entry to "1180                   << "function " << *PreviousFunction << '\n';1181 1182        registerName(0);1183 1184        PreviousFunction->addEntryPointAtOffset(SymbolAddress -1185                                                PreviousFunction->getAddress());1186 1187        // Remove the symbol from FileSymRefs so that we can skip it from1188        // in the future.1189        auto SI = llvm::find_if(1190            llvm::make_range(FileSymRefs.equal_range(SymbolAddress)),1191            [&](auto SymIt) { return SymIt.second == Symbol; });1192        assert(SI != FileSymRefs.end() && "symbol expected to be present");1193        assert(SI->second == Symbol && "wrong symbol found");1194        FileSymRefs.erase(SI);1195      }1196      continue;1197    }1198 1199    // Checkout for conflicts with function data from FDEs.1200    bool IsSimple = true;1201    auto FDEI = CFIRdWrt->getFDEs().lower_bound(SymbolAddress);1202    if (FDEI != CFIRdWrt->getFDEs().end()) {1203      const dwarf::FDE &FDE = *FDEI->second;1204      if (FDEI->first != SymbolAddress) {1205        // There's no matching starting address in FDE. Make sure the previous1206        // FDE does not contain this address.1207        if (FDEI != CFIRdWrt->getFDEs().begin()) {1208          --FDEI;1209          const dwarf::FDE &PrevFDE = *FDEI->second;1210          uint64_t PrevStart = PrevFDE.getInitialLocation();1211          uint64_t PrevLength = PrevFDE.getAddressRange();1212          if (SymbolAddress > PrevStart &&1213              SymbolAddress < PrevStart + PrevLength) {1214            BC->errs() << "BOLT-ERROR: function " << UniqueName1215                       << " is in conflict with FDE ["1216                       << Twine::utohexstr(PrevStart) << ", "1217                       << Twine::utohexstr(PrevStart + PrevLength)1218                       << "). Skipping.\n";1219            IsSimple = false;1220          }1221        }1222      } else if (FDE.getAddressRange() != SymbolSize) {1223        if (SymbolSize) {1224          // Function addresses match but sizes differ.1225          BC->errs() << "BOLT-WARNING: sizes differ for function " << UniqueName1226                     << ". FDE : " << FDE.getAddressRange()1227                     << "; symbol table : " << SymbolSize1228                     << ". Using max size.\n";1229        }1230        SymbolSize = std::max(SymbolSize, FDE.getAddressRange());1231        if (BC->getBinaryDataAtAddress(SymbolAddress)) {1232          BC->setBinaryDataSize(SymbolAddress, SymbolSize);1233        } else {1234          LLVM_DEBUG(dbgs() << "BOLT-DEBUG: No BD @ 0x"1235                            << Twine::utohexstr(SymbolAddress) << "\n");1236        }1237      }1238    }1239 1240    BinaryFunction *BF = nullptr;1241    // Since function may not have yet obtained its real size, do a search1242    // using the list of registered functions instead of calling1243    // getBinaryFunctionAtAddress().1244    auto BFI = BC->getBinaryFunctions().find(SymbolAddress);1245    if (BFI != BC->getBinaryFunctions().end()) {1246      BF = &BFI->second;1247      // Duplicate the function name. Make sure everything matches before we add1248      // an alternative name.1249      if (SymbolSize != BF->getSize()) {1250        if (opts::Verbosity >= 1) {1251          if (SymbolSize && BF->getSize())1252            BC->errs() << "BOLT-WARNING: size mismatch for duplicate entries "1253                       << *BF << " and " << UniqueName << '\n';1254          BC->outs() << "BOLT-INFO: adjusting size of function " << *BF1255                     << " old " << BF->getSize() << " new " << SymbolSize1256                     << "\n";1257        }1258        BF->setSize(std::max(SymbolSize, BF->getSize()));1259        BC->setBinaryDataSize(SymbolAddress, BF->getSize());1260      }1261      BF->addAlternativeName(UniqueName);1262    } else {1263      ErrorOr<BinarySection &> Section =1264          BC->getSectionForAddress(SymbolAddress);1265      // Skip symbols from invalid sections1266      if (!Section) {1267        BC->errs() << "BOLT-WARNING: " << UniqueName << " (0x"1268                   << Twine::utohexstr(SymbolAddress)1269                   << ") does not have any section\n";1270        continue;1271      }1272 1273      // Skip symbols from zero-sized sections.1274      if (!Section->getSize())1275        continue;1276 1277      BF = BC->createBinaryFunction(UniqueName, *Section, SymbolAddress,1278                                    SymbolSize);1279      if (!IsSimple)1280        BF->setSimple(false);1281    }1282 1283    // Check if it's a cold function fragment.1284    if (FunctionFragmentTemplate.match(SymName)) {1285      static bool PrintedWarning = false;1286      if (!PrintedWarning) {1287        PrintedWarning = true;1288        BC->errs() << "BOLT-WARNING: split function detected on input : "1289                   << SymName;1290        if (BC->HasRelocations)1291          BC->errs() << ". The support is limited in relocation mode\n";1292        else1293          BC->errs() << '\n';1294      }1295      BC->HasSplitFunctions = true;1296      BF->IsFragment = true;1297    }1298 1299    if (!AlternativeName.empty())1300      BF->addAlternativeName(AlternativeName);1301 1302    registerName(SymbolSize);1303    PreviousFunction = BF;1304  }1305 1306  // Read dynamic relocation first as their presence affects the way we process1307  // static relocations. E.g. we will ignore a static relocation at an address1308  // that is a subject to dynamic relocation processing.1309  processDynamicRelocations();1310 1311  // Process PLT section.1312  disassemblePLT();1313 1314  // See if we missed any functions marked by FDE.1315  for (const auto &FDEI : CFIRdWrt->getFDEs()) {1316    const uint64_t Address = FDEI.first;1317    const dwarf::FDE *FDE = FDEI.second;1318    const BinaryFunction *BF = BC->getBinaryFunctionAtAddress(Address);1319    if (BF)1320      continue;1321 1322    BF = BC->getBinaryFunctionContainingAddress(Address);1323    if (BF) {1324      BC->errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Address)1325                 << ", 0x" << Twine::utohexstr(Address + FDE->getAddressRange())1326                 << ") conflicts with function " << *BF << '\n';1327      continue;1328    }1329 1330    if (opts::Verbosity >= 1)1331      BC->errs() << "BOLT-WARNING: FDE [0x" << Twine::utohexstr(Address)1332                 << ", 0x" << Twine::utohexstr(Address + FDE->getAddressRange())1333                 << ") has no corresponding symbol table entry\n";1334 1335    ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);1336    assert(Section && "cannot get section for address from FDE");1337    std::string FunctionName =1338        "__BOLT_FDE_FUNCat" + Twine::utohexstr(Address).str();1339    BC->createBinaryFunction(FunctionName, *Section, Address,1340                             FDE->getAddressRange());1341  }1342 1343  BC->setHasSymbolsWithFileName(FileSymbols.size());1344 1345  // Now that all the functions were created - adjust their boundaries.1346  adjustFunctionBoundaries(MarkerSymbols);1347 1348  // Annotate functions with code/data markers in AArch641349  for (auto &[Address, Type] : MarkerSymbols) {1350    auto *BF = BC->getBinaryFunctionContainingAddress(Address,1351                                                      /*CheckPastEnd*/ false,1352                                                      /*UseMaxSize*/ true);1353 1354    if (!BF) {1355      // Stray marker1356      continue;1357    }1358    const auto EntryOffset = Address - BF->getAddress();1359    if (Type == MarkerSymType::CODE) {1360      BF->markCodeAtOffset(EntryOffset);1361      continue;1362    }1363    if (Type == MarkerSymType::DATA) {1364      BF->markDataAtOffset(EntryOffset);1365      BC->AddressToConstantIslandMap[Address] = BF;1366      continue;1367    }1368    llvm_unreachable("Unknown marker");1369  }1370 1371  if (BC->isAArch64()) {1372    // Check for dynamic relocations that might be contained in1373    // constant islands.1374    for (const BinarySection &Section : BC->allocatableSections()) {1375      const uint64_t SectionAddress = Section.getAddress();1376      for (const Relocation &Rel : Section.dynamicRelocations()) {1377        const uint64_t RelAddress = SectionAddress + Rel.Offset;1378        BinaryFunction *BF =1379            BC->getBinaryFunctionContainingAddress(RelAddress,1380                                                   /*CheckPastEnd*/ false,1381                                                   /*UseMaxSize*/ true);1382        if (BF) {1383          assert(Rel.isRelative() && "Expected relative relocation for island");1384          BC->logBOLTErrorsAndQuitOnFatal(1385              BF->markIslandDynamicRelocationAtAddress(RelAddress));1386        }1387      }1388    }1389 1390    // The linker may omit data markers for absolute long veneers. Introduce1391    // those markers artificially to assist the disassembler.1392    for (BinaryFunction &BF :1393         llvm::make_second_range(BC->getBinaryFunctions())) {1394      if (BF.getOneName().starts_with("__AArch64AbsLongThunk_") &&1395          BF.getSize() == 16 && !BF.getSizeOfDataInCodeAt(8)) {1396        BC->errs() << "BOLT-WARNING: missing data marker detected in veneer "1397                   << BF << '\n';1398        BF.markDataAtOffset(8);1399        BC->AddressToConstantIslandMap[BF.getAddress() + 8] = &BF;1400      }1401    }1402  }1403 1404  if (!BC->IsLinuxKernel) {1405    // Read all relocations now that we have binary functions mapped.1406    processRelocations();1407  }1408 1409  registerFragments();1410  FileSymbols.clear();1411  FileSymRefs.clear();1412 1413  discoverBOLTReserved();1414}1415 1416void RewriteInstance::discoverBOLTReserved() {1417  BinaryData *StartBD = BC->getBinaryDataByName(getBOLTReservedStart());1418  BinaryData *EndBD = BC->getBinaryDataByName(getBOLTReservedEnd());1419  if (!StartBD != !EndBD) {1420    BC->errs() << "BOLT-ERROR: one of the symbols is missing from the binary: "1421               << getBOLTReservedStart() << ", " << getBOLTReservedEnd()1422               << '\n';1423    exit(1);1424  }1425 1426  if (!StartBD)1427    return;1428 1429  if (StartBD->getAddress() >= EndBD->getAddress()) {1430    BC->errs() << "BOLT-ERROR: invalid reserved space boundaries\n";1431    exit(1);1432  }1433  BC->BOLTReserved = AddressRange(StartBD->getAddress(), EndBD->getAddress());1434  BC->outs() << "BOLT-INFO: using reserved space for allocating new sections\n";1435 1436  PHDRTableOffset = 0;1437  PHDRTableAddress = 0;1438  NewTextSegmentAddress = 0;1439  NewTextSegmentOffset = 0;1440  NextAvailableAddress = BC->BOLTReserved.start();1441}1442 1443Error RewriteInstance::discoverRtInitAddress() {1444  if (BC->HasInterpHeader && opts::RuntimeLibInitHook == opts::RLIH_ENTRY_POINT)1445    return Error::success();1446 1447  // Use DT_INIT if it's available.1448  if (BC->InitAddress && opts::RuntimeLibInitHook <= opts::RLIH_INIT) {1449    BC->StartFunctionAddress = BC->InitAddress;1450    return Error::success();1451  }1452 1453  if (!BC->InitArrayAddress || !BC->InitArraySize) {1454    return createStringError(std::errc::not_supported,1455                             "Instrumentation of shared library needs either "1456                             "DT_INIT or DT_INIT_ARRAY");1457  }1458 1459  if (*BC->InitArraySize < BC->AsmInfo->getCodePointerSize()) {1460    return createStringError(std::errc::not_supported,1461                             "Need at least 1 DT_INIT_ARRAY slot");1462  }1463 1464  ErrorOr<BinarySection &> InitArraySection =1465      BC->getSectionForAddress(*BC->InitArrayAddress);1466  if (auto EC = InitArraySection.getError())1467    return errorCodeToError(EC);1468 1469  if (InitArraySection->getAddress() != *BC->InitArrayAddress) {1470    return createStringError(std::errc::not_supported,1471                             "Inconsistent address of .init_array section");1472  }1473 1474  if (const Relocation *Reloc = InitArraySection->getDynamicRelocationAt(0)) {1475    if (Reloc->isRelative()) {1476      BC->StartFunctionAddress = Reloc->Addend;1477    } else {1478      MCSymbol *Sym = Reloc->Symbol;1479      if (!Sym)1480        return createStringError(1481            std::errc::not_supported,1482            "Failed to locate symbol for 0 entry of .init_array");1483      const BinaryFunction *BF = BC->getFunctionForSymbol(Sym);1484      if (!BF)1485        return createStringError(1486            std::errc::not_supported,1487            "Failed to locate binary function for 0 entry of .init_array");1488      BC->StartFunctionAddress = BF->getAddress() + Reloc->Addend;1489    }1490    return Error::success();1491  }1492 1493  if (const Relocation *Reloc = InitArraySection->getRelocationAt(0)) {1494    BC->StartFunctionAddress = Reloc->Value;1495    return Error::success();1496  }1497 1498  return createStringError(std::errc::not_supported,1499                           "No relocation for first DT_INIT_ARRAY slot");1500}1501 1502Error RewriteInstance::discoverRtFiniAddress() {1503  // Use DT_FINI if it's available.1504  if (BC->FiniAddress) {1505    BC->FiniFunctionAddress = BC->FiniAddress;1506    return Error::success();1507  }1508 1509  if (!BC->FiniArrayAddress || !BC->FiniArraySize) {1510    return createStringError(1511        std::errc::not_supported,1512        "Instrumentation needs either DT_FINI or DT_FINI_ARRAY");1513  }1514 1515  if (*BC->FiniArraySize < BC->AsmInfo->getCodePointerSize()) {1516    return createStringError(std::errc::not_supported,1517                             "Need at least 1 DT_FINI_ARRAY slot");1518  }1519 1520  ErrorOr<BinarySection &> FiniArraySection =1521      BC->getSectionForAddress(*BC->FiniArrayAddress);1522  if (auto EC = FiniArraySection.getError())1523    return errorCodeToError(EC);1524 1525  if (FiniArraySection->getAddress() != *BC->FiniArrayAddress) {1526    return createStringError(std::errc::not_supported,1527                             "Inconsistent address of .fini_array section");1528  }1529 1530  if (const Relocation *Reloc = FiniArraySection->getDynamicRelocationAt(0)) {1531    BC->FiniFunctionAddress = Reloc->Addend;1532    return Error::success();1533  }1534 1535  if (const Relocation *Reloc = FiniArraySection->getRelocationAt(0)) {1536    BC->FiniFunctionAddress = Reloc->Value;1537    return Error::success();1538  }1539 1540  return createStringError(std::errc::not_supported,1541                           "No relocation for first DT_FINI_ARRAY slot");1542}1543 1544Error RewriteInstance::updateRtInitReloc() {1545  if (BC->HasInterpHeader && opts::RuntimeLibInitHook == opts::RLIH_ENTRY_POINT)1546    return Error::success();1547 1548  // Updating DT_INIT is handled by patchELFDynamic.1549  if (BC->InitAddress && opts::RuntimeLibInitHook <= opts::RLIH_INIT)1550    return Error::success();1551 1552  const RuntimeLibrary *RT = BC->getRuntimeLibrary();1553  if (!RT || !RT->getRuntimeStartAddress())1554    return Error::success();1555 1556  if (!BC->InitArrayAddress)1557    return Error::success();1558 1559  if (!BC->InitArrayAddress || !BC->InitArraySize)1560    return createStringError(std::errc::not_supported,1561                             "inconsistent .init_array state");1562 1563  ErrorOr<BinarySection &> InitArraySection =1564      BC->getSectionForAddress(*BC->InitArrayAddress);1565  if (!InitArraySection)1566    return createStringError(std::errc::not_supported, ".init_array removed");1567 1568  if (std::optional<Relocation> Reloc =1569          InitArraySection->takeDynamicRelocationAt(0)) {1570    if (Reloc->isRelative()) {1571      if (Reloc->Addend != BC->StartFunctionAddress)1572        return createStringError(std::errc::not_supported,1573                                 "inconsistent .init_array dynamic relocation");1574      Reloc->Addend = RT->getRuntimeStartAddress();1575      InitArraySection->addDynamicRelocation(*Reloc);1576    } else {1577      MCSymbol *Sym = Reloc->Symbol;1578      if (!Sym)1579        return createStringError(1580            std::errc::not_supported,1581            "Failed to locate symbol for 0 entry of .init_array");1582      const BinaryFunction *BF = BC->getFunctionForSymbol(Sym);1583      if (!BF)1584        return createStringError(1585            std::errc::not_supported,1586            "Failed to locate binary function for 0 entry of .init_array");1587      if (BF->getAddress() + Reloc->Addend != BC->StartFunctionAddress)1588        return createStringError(std::errc::not_supported,1589                                 "inconsistent .init_array dynamic relocation");1590      InitArraySection->addDynamicRelocation(Relocation{1591          /*Offset*/ 0, /*Symbol*/ nullptr, /*Type*/ Relocation::getAbs64(),1592          /*Addend*/ RT->getRuntimeStartAddress(), /*Value*/ 0});1593    }1594  }1595  // Update the static relocation by adding a pending relocation which will get1596  // patched when flushPendingRelocations is called in rewriteFile. Note that1597  // flushPendingRelocations will calculate the value to patch as1598  // "Symbol + Addend". Since we don't have a symbol, just set the addend to the1599  // desired value.1600  InitArraySection->addPendingRelocation(Relocation{1601      /*Offset*/ 0, /*Symbol*/ nullptr, /*Type*/ Relocation::getAbs64(),1602      /*Addend*/ RT->getRuntimeStartAddress(), /*Value*/ 0});1603  BC->outs()1604      << "BOLT-INFO: runtime library initialization was hooked via .init_array "1605         "entry, set to 0x"1606      << Twine::utohexstr(RT->getRuntimeStartAddress()) << "\n";1607  return Error::success();1608}1609 1610Error RewriteInstance::updateRtFiniReloc() {1611  // Updating DT_FINI is handled by patchELFDynamic.1612  if (BC->FiniAddress)1613    return Error::success();1614 1615  const RuntimeLibrary *RT = BC->getRuntimeLibrary();1616  if (!RT || !RT->getRuntimeFiniAddress())1617    return Error::success();1618 1619  if (!BC->FiniArrayAddress || !BC->FiniArraySize)1620    return createStringError(std::errc::not_supported,1621                             "inconsistent .fini_array state");1622 1623  ErrorOr<BinarySection &> FiniArraySection =1624      BC->getSectionForAddress(*BC->FiniArrayAddress);1625  if (!FiniArraySection)1626    return createStringError(std::errc::not_supported, ".fini_array removed");1627 1628  if (std::optional<Relocation> Reloc =1629          FiniArraySection->takeDynamicRelocationAt(0)) {1630    if (Reloc->Addend != BC->FiniFunctionAddress)1631      return createStringError(std::errc::not_supported,1632                               "inconsistent .fini_array dynamic relocation");1633    Reloc->Addend = RT->getRuntimeFiniAddress();1634    FiniArraySection->addDynamicRelocation(*Reloc);1635  }1636 1637  // Update the static relocation by adding a pending relocation which will get1638  // patched when flushPendingRelocations is called in rewriteFile. Note that1639  // flushPendingRelocations will calculate the value to patch as1640  // "Symbol + Addend". Since we don't have a symbol, just set the addend to the1641  // desired value.1642  FiniArraySection->addPendingRelocation(Relocation{1643      /*Offset*/ 0, /*Symbol*/ nullptr, /*Type*/ Relocation::getAbs64(),1644      /*Addend*/ RT->getRuntimeFiniAddress(), /*Value*/ 0});1645  BC->outs() << "BOLT-INFO: runtime library finalization was hooked via "1646                ".fini_array entry, set to 0x"1647             << Twine::utohexstr(RT->getRuntimeFiniAddress()) << "\n";1648  return Error::success();1649}1650 1651void RewriteInstance::registerFragments() {1652  if (!BC->HasSplitFunctions ||1653      opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive)1654    return;1655 1656  // Process fragments with ambiguous parents separately as they are typically a1657  // vanishing minority of cases and require expensive symbol table lookups.1658  std::vector<std::pair<StringRef, BinaryFunction *>> AmbiguousFragments;1659  for (auto &BFI : BC->getBinaryFunctions()) {1660    BinaryFunction &Function = BFI.second;1661    if (!Function.isFragment())1662      continue;1663    for (StringRef Name : Function.getNames()) {1664      StringRef BaseName = NR.restore(Name);1665      const bool IsGlobal = BaseName == Name;1666      SmallVector<StringRef> Matches;1667      if (!FunctionFragmentTemplate.match(BaseName, &Matches))1668        continue;1669      StringRef ParentName = Matches[1];1670      const BinaryData *BD = BC->getBinaryDataByName(ParentName);1671      const uint64_t NumPossibleLocalParents =1672          NR.getUniquifiedNameCount(ParentName);1673      // The most common case: single local parent fragment.1674      if (!BD && NumPossibleLocalParents == 1) {1675        BD = BC->getBinaryDataByName(NR.getUniqueName(ParentName, 1));1676      } else if (BD && (!NumPossibleLocalParents || IsGlobal)) {1677        // Global parent and either no local candidates (second most common), or1678        // the fragment is global as well (uncommon).1679      } else {1680        // Any other case: need to disambiguate using FILE symbols.1681        AmbiguousFragments.emplace_back(ParentName, &Function);1682        continue;1683      }1684      if (BD) {1685        BinaryFunction *BF = BC->getFunctionForSymbol(BD->getSymbol());1686        if (BF == &Function) {1687          BC->errs()1688              << "BOLT-WARNING: fragment maps to the same function as parent: "1689              << Function << '\n';1690          continue;1691        }1692        if (BF) {1693          BC->registerFragment(Function, *BF);1694          continue;1695        }1696      }1697      BC->errs() << "BOLT-ERROR: parent function not found for " << Function1698                 << '\n';1699      exit(1);1700    }1701  }1702 1703  if (AmbiguousFragments.empty())1704    return;1705 1706  if (!BC->hasSymbolsWithFileName()) {1707    BC->errs() << "BOLT-ERROR: input file has split functions but does not "1708                  "have FILE symbols. If the binary was stripped, preserve "1709                  "FILE symbols with --keep-file-symbols strip option\n";1710    exit(1);1711  }1712 1713  // The first global symbol is identified by the symbol table sh_info value.1714  // Used as local symbol search stopping point.1715  auto *ELF64LEFile = cast<ELF64LEObjectFile>(InputFile);1716  const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();1717  auto *SymTab = llvm::find_if(cantFail(Obj.sections()), [](const auto &Sec) {1718    return Sec.sh_type == ELF::SHT_SYMTAB;1719  });1720  assert(SymTab);1721  // Symtab sh_info contains the value one greater than the symbol table index1722  // of the last local symbol.1723  ELFSymbolRef LocalSymEnd = ELF64LEFile->toSymbolRef(SymTab, SymTab->sh_info);1724 1725  for (auto &Fragment : AmbiguousFragments) {1726    const StringRef &ParentName = Fragment.first;1727    BinaryFunction *BF = Fragment.second;1728    const uint64_t Address = BF->getAddress();1729 1730    // Get fragment's own symbol1731    const auto SymIt = llvm::find_if(1732        llvm::make_range(FileSymRefs.equal_range(Address)), [&](auto SI) {1733          StringRef Name = cantFail(SI.second.getName());1734          return Name.contains(ParentName);1735        });1736    if (SymIt == FileSymRefs.end()) {1737      BC->errs()1738          << "BOLT-ERROR: symbol lookup failed for function at address 0x"1739          << Twine::utohexstr(Address) << '\n';1740      exit(1);1741    }1742 1743    // Find containing FILE symbol1744    ELFSymbolRef Symbol = SymIt->second;1745    auto FSI = llvm::upper_bound(FileSymbols, Symbol);1746    if (FSI == FileSymbols.begin()) {1747      BC->errs() << "BOLT-ERROR: owning FILE symbol not found for symbol "1748                 << cantFail(Symbol.getName()) << '\n';1749      exit(1);1750    }1751 1752    ELFSymbolRef StopSymbol = LocalSymEnd;1753    if (FSI != FileSymbols.end())1754      StopSymbol = *FSI;1755 1756    uint64_t ParentAddress{0};1757 1758    // Check if containing FILE symbol is BOLT emitted synthetic symbol marking1759    // local fragments of global parents.1760    if (cantFail(FSI[-1].getName()) == getBOLTFileSymbolName())1761      goto registerParent;1762 1763    // BOLT split fragment symbols are emitted just before the main function1764    // symbol.1765    for (ELFSymbolRef NextSymbol = Symbol; NextSymbol < StopSymbol;1766         NextSymbol.moveNext()) {1767      StringRef Name = cantFail(NextSymbol.getName());1768      if (Name == ParentName) {1769        ParentAddress = cantFail(NextSymbol.getValue());1770        goto registerParent;1771      }1772      if (Name.starts_with(ParentName))1773        // With multi-way splitting, there are multiple fragments with different1774        // suffixes. Parent follows the last fragment.1775        continue;1776      break;1777    }1778 1779    // Iterate over local file symbols and check symbol names to match parent.1780    for (ELFSymbolRef Symbol(FSI[-1]); Symbol < StopSymbol; Symbol.moveNext()) {1781      if (cantFail(Symbol.getName()) == ParentName) {1782        ParentAddress = cantFail(Symbol.getAddress());1783        break;1784      }1785    }1786 1787registerParent:1788    // No local parent is found, use global parent function.1789    if (!ParentAddress)1790      if (BinaryData *ParentBD = BC->getBinaryDataByName(ParentName))1791        ParentAddress = ParentBD->getAddress();1792 1793    if (BinaryFunction *ParentBF =1794            BC->getBinaryFunctionAtAddress(ParentAddress)) {1795      BC->registerFragment(*BF, *ParentBF);1796      continue;1797    }1798    BC->errs() << "BOLT-ERROR: parent function not found for " << *BF << '\n';1799    exit(1);1800  }1801}1802 1803void RewriteInstance::createPLTBinaryFunction(uint64_t TargetAddress,1804                                              uint64_t EntryAddress,1805                                              uint64_t EntrySize) {1806  if (!TargetAddress)1807    return;1808 1809  auto setPLTSymbol = [&](BinaryFunction *BF, StringRef Name) {1810    const unsigned PtrSize = BC->AsmInfo->getCodePointerSize();1811    MCSymbol *TargetSymbol = BC->registerNameAtAddress(1812        Name.str() + "@GOT", TargetAddress, PtrSize, PtrSize);1813    BF->setPLTSymbol(TargetSymbol);1814  };1815 1816  BinaryFunction *BF = BC->getBinaryFunctionAtAddress(EntryAddress);1817  if (BF && BC->isAArch64()) {1818    // Handle IFUNC trampoline with symbol1819    setPLTSymbol(BF, BF->getOneName());1820    return;1821  }1822 1823  const Relocation *Rel = BC->getDynamicRelocationAt(TargetAddress);1824  if (!Rel)1825    return;1826 1827  MCSymbol *Symbol = Rel->Symbol;1828  if (!Symbol) {1829    if (BC->isRISCV() || !Rel->Addend || !Rel->isIRelative())1830      return;1831 1832    // IFUNC trampoline without symbol1833    BinaryFunction *TargetBF = BC->getBinaryFunctionAtAddress(Rel->Addend);1834    if (!TargetBF) {1835      BC->errs()1836          << "BOLT-WARNING: Expected BF to be presented as IFUNC resolver at "1837          << Twine::utohexstr(Rel->Addend) << ", skipping\n";1838      return;1839    }1840 1841    Symbol = TargetBF->getSymbol();1842  }1843 1844  ErrorOr<BinarySection &> Section = BC->getSectionForAddress(EntryAddress);1845  assert(Section && "cannot get section for address");1846  if (!BF)1847    BF = BC->createBinaryFunction(Symbol->getName().str() + "@PLT", *Section,1848                                  EntryAddress, 0, EntrySize,1849                                  Section->getAlignment());1850  else1851    BF->addAlternativeName(Symbol->getName().str() + "@PLT");1852  setPLTSymbol(BF, Symbol->getName());1853}1854 1855void RewriteInstance::disassemblePLTInstruction(const BinarySection &Section,1856                                                uint64_t InstrOffset,1857                                                MCInst &Instruction,1858                                                uint64_t &InstrSize) {1859  const uint64_t SectionAddress = Section.getAddress();1860  const uint64_t SectionSize = Section.getSize();1861  StringRef PLTContents = Section.getContents();1862  ArrayRef<uint8_t> PLTData(1863      reinterpret_cast<const uint8_t *>(PLTContents.data()), SectionSize);1864 1865  const uint64_t InstrAddr = SectionAddress + InstrOffset;1866  if (!BC->DisAsm->getInstruction(Instruction, InstrSize,1867                                  PLTData.slice(InstrOffset), InstrAddr,1868                                  nulls())) {1869    BC->errs()1870        << "BOLT-ERROR: unable to disassemble instruction in PLT section "1871        << Section.getName() << formatv(" at offset {0:x}\n", InstrOffset);1872    exit(1);1873  }1874}1875 1876void RewriteInstance::disassemblePLTSectionAArch64(BinarySection &Section) {1877  const uint64_t SectionAddress = Section.getAddress();1878  const uint64_t SectionSize = Section.getSize();1879 1880  uint64_t InstrOffset = 0;1881  // Locate new plt entry1882  while (InstrOffset < SectionSize) {1883    InstructionListType Instructions;1884    MCInst Instruction;1885    uint64_t EntryOffset = InstrOffset;1886    uint64_t EntrySize = 0;1887    uint64_t InstrSize;1888    // Loop through entry instructions1889    while (InstrOffset < SectionSize) {1890      disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);1891      EntrySize += InstrSize;1892      if (!BC->MIB->isIndirectBranch(Instruction)) {1893        Instructions.emplace_back(Instruction);1894        InstrOffset += InstrSize;1895        continue;1896      }1897 1898      const uint64_t EntryAddress = SectionAddress + EntryOffset;1899      const uint64_t TargetAddress = BC->MIB->analyzePLTEntry(1900          Instruction, Instructions.begin(), Instructions.end(), EntryAddress);1901 1902      createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize);1903      break;1904    }1905 1906    // Branch instruction1907    InstrOffset += InstrSize;1908 1909    // Skip nops if any1910    while (InstrOffset < SectionSize) {1911      disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);1912      if (!BC->MIB->isNoop(Instruction))1913        break;1914 1915      InstrOffset += InstrSize;1916    }1917  }1918}1919 1920void RewriteInstance::disassemblePLTSectionRISCV(BinarySection &Section) {1921  const uint64_t SectionAddress = Section.getAddress();1922  const uint64_t SectionSize = Section.getSize();1923  StringRef PLTContents = Section.getContents();1924  ArrayRef<uint8_t> PLTData(1925      reinterpret_cast<const uint8_t *>(PLTContents.data()), SectionSize);1926 1927  auto disassembleInstruction = [&](uint64_t InstrOffset, MCInst &Instruction,1928                                    uint64_t &InstrSize) {1929    const uint64_t InstrAddr = SectionAddress + InstrOffset;1930    if (!BC->DisAsm->getInstruction(Instruction, InstrSize,1931                                    PLTData.slice(InstrOffset), InstrAddr,1932                                    nulls())) {1933      BC->errs()1934          << "BOLT-ERROR: unable to disassemble instruction in PLT section "1935          << Section.getName() << " at offset 0x"1936          << Twine::utohexstr(InstrOffset) << '\n';1937      exit(1);1938    }1939  };1940 1941  // Skip the first special entry since no relocation points to it.1942  uint64_t InstrOffset = 32;1943 1944  while (InstrOffset < SectionSize) {1945    InstructionListType Instructions;1946    MCInst Instruction;1947    const uint64_t EntryOffset = InstrOffset;1948    const uint64_t EntrySize = 16;1949    uint64_t InstrSize;1950 1951    while (InstrOffset < EntryOffset + EntrySize) {1952      disassembleInstruction(InstrOffset, Instruction, InstrSize);1953      Instructions.emplace_back(Instruction);1954      InstrOffset += InstrSize;1955    }1956 1957    const uint64_t EntryAddress = SectionAddress + EntryOffset;1958    const uint64_t TargetAddress = BC->MIB->analyzePLTEntry(1959        Instruction, Instructions.begin(), Instructions.end(), EntryAddress);1960 1961    createPLTBinaryFunction(TargetAddress, EntryAddress, EntrySize);1962  }1963}1964 1965void RewriteInstance::disassemblePLTSectionX86(BinarySection &Section,1966                                               uint64_t EntrySize) {1967  const uint64_t SectionAddress = Section.getAddress();1968  const uint64_t SectionSize = Section.getSize();1969 1970  for (uint64_t EntryOffset = 0; EntryOffset + EntrySize <= SectionSize;1971       EntryOffset += EntrySize) {1972    MCInst Instruction;1973    uint64_t InstrSize, InstrOffset = EntryOffset;1974    while (InstrOffset < EntryOffset + EntrySize) {1975      disassemblePLTInstruction(Section, InstrOffset, Instruction, InstrSize);1976      // Check if the entry size needs adjustment.1977      if (EntryOffset == 0 && BC->MIB->isTerminateBranch(Instruction) &&1978          EntrySize == 8)1979        EntrySize = 16;1980 1981      if (BC->MIB->isIndirectBranch(Instruction))1982        break;1983 1984      InstrOffset += InstrSize;1985    }1986 1987    if (InstrOffset + InstrSize > EntryOffset + EntrySize)1988      continue;1989 1990    uint64_t TargetAddress;1991    if (!BC->MIB->evaluateMemOperandTarget(Instruction, TargetAddress,1992                                           SectionAddress + InstrOffset,1993                                           InstrSize)) {1994      BC->errs() << "BOLT-ERROR: error evaluating PLT instruction at offset 0x"1995                 << Twine::utohexstr(SectionAddress + InstrOffset) << '\n';1996      exit(1);1997    }1998 1999    createPLTBinaryFunction(TargetAddress, SectionAddress + EntryOffset,2000                            EntrySize);2001  }2002}2003 2004void RewriteInstance::disassemblePLT() {2005  auto analyzeOnePLTSection = [&](BinarySection &Section, uint64_t EntrySize) {2006    if (BC->isAArch64())2007      return disassemblePLTSectionAArch64(Section);2008    if (BC->isRISCV())2009      return disassemblePLTSectionRISCV(Section);2010    if (BC->isX86())2011      return disassemblePLTSectionX86(Section, EntrySize);2012    llvm_unreachable("Unmplemented PLT");2013  };2014 2015  for (BinarySection &Section : BC->allocatableSections()) {2016    const PLTSectionInfo *PLTSI = getPLTSectionInfo(Section.getName());2017    if (!PLTSI)2018      continue;2019 2020    analyzeOnePLTSection(Section, PLTSI->EntrySize);2021 2022    BinaryFunction *PltBF;2023    auto BFIter = BC->getBinaryFunctions().find(Section.getAddress());2024    if (BFIter != BC->getBinaryFunctions().end()) {2025      PltBF = &BFIter->second;2026    } else {2027      // If we did not register any function at the start of the section,2028      // then it must be a general PLT entry. Add a function at the location.2029      PltBF = BC->createBinaryFunction(2030          "__BOLT_PSEUDO_" + Section.getName().str(), Section,2031          Section.getAddress(), 0, PLTSI->EntrySize, Section.getAlignment());2032    }2033    PltBF->setPseudo(true);2034  }2035}2036 2037void RewriteInstance::adjustFunctionBoundaries(2038    DenseMap<uint64_t, MarkerSymType> &MarkerSyms) {2039  for (auto BFI = BC->getBinaryFunctions().begin(),2040            BFE = BC->getBinaryFunctions().end();2041       BFI != BFE; ++BFI) {2042    BinaryFunction &Function = BFI->second;2043    const BinaryFunction *NextFunction = nullptr;2044    if (std::next(BFI) != BFE)2045      NextFunction = &std::next(BFI)->second;2046 2047    // Check if there's a symbol or a function with a larger address in the2048    // same section. If there is - it determines the maximum size for the2049    // current function. Otherwise, it is the size of a containing section2050    // the defines it.2051    //2052    // NOTE: ignore some symbols that could be tolerated inside the body2053    //       of a function.2054    auto NextSymRefI = FileSymRefs.upper_bound(Function.getAddress());2055    while (NextSymRefI != FileSymRefs.end()) {2056      SymbolRef &Symbol = NextSymRefI->second;2057      const uint64_t SymbolAddress = NextSymRefI->first;2058      const uint64_t SymbolSize = ELFSymbolRef(Symbol).getSize();2059 2060      if (NextFunction && SymbolAddress >= NextFunction->getAddress())2061        break;2062 2063      if (!Function.isSymbolValidInScope(Symbol, SymbolSize))2064        break;2065 2066      // Skip basic block labels. This happens on RISC-V with linker relaxation2067      // enabled because every branch needs a relocation and corresponding2068      // symbol. We don't want to add such symbols as entry points.2069      const auto PrivateLabelPrefix = BC->AsmInfo->getPrivateLabelPrefix();2070      if (!PrivateLabelPrefix.empty() &&2071          cantFail(Symbol.getName()).starts_with(PrivateLabelPrefix)) {2072        ++NextSymRefI;2073        continue;2074      }2075 2076      auto It = MarkerSyms.find(NextSymRefI->first);2077      if (It == MarkerSyms.end() || It->second != MarkerSymType::DATA) {2078        // This is potentially another entry point into the function.2079        uint64_t EntryOffset = NextSymRefI->first - Function.getAddress();2080        LLVM_DEBUG(dbgs() << "BOLT-DEBUG: adding entry point to function "2081                          << Function << " at offset 0x"2082                          << Twine::utohexstr(EntryOffset) << '\n');2083        Function.addEntryPointAtOffset(EntryOffset);2084      }2085 2086      ++NextSymRefI;2087    }2088 2089    // Function runs at most till the end of the containing section.2090    uint64_t NextObjectAddress = Function.getOriginSection()->getEndAddress();2091    // Or till the next object marked by a symbol.2092    if (NextSymRefI != FileSymRefs.end())2093      NextObjectAddress = std::min(NextSymRefI->first, NextObjectAddress);2094 2095    // Or till the next function not marked by a symbol.2096    if (NextFunction)2097      NextObjectAddress =2098          std::min(NextFunction->getAddress(), NextObjectAddress);2099 2100    const uint64_t MaxSize = NextObjectAddress - Function.getAddress();2101    if (MaxSize < Function.getSize()) {2102      BC->errs() << "BOLT-ERROR: symbol seen in the middle of the function "2103                 << Function << ". Skipping.\n";2104      Function.setSimple(false);2105      Function.setMaxSize(Function.getSize());2106      continue;2107    }2108    Function.setMaxSize(MaxSize);2109    if (!Function.getSize() && Function.isSimple()) {2110      // Some assembly functions have their size set to 0, use the max2111      // size as their real size.2112      if (opts::Verbosity >= 1)2113        BC->outs() << "BOLT-INFO: setting size of function " << Function2114                   << " to " << Function.getMaxSize() << " (was 0)\n";2115      Function.setSize(Function.getMaxSize());2116    }2117  }2118}2119 2120void RewriteInstance::relocateEHFrameSection() {2121  assert(EHFrameSection && "Non-empty .eh_frame section expected.");2122 2123  BinarySection *RelocatedEHFrameSection =2124      getSection(".relocated" + getEHFrameSectionName());2125  assert(RelocatedEHFrameSection &&2126         "Relocated eh_frame section should be preregistered.");2127  DWARFDataExtractor DE(EHFrameSection->getContents(),2128                        BC->AsmInfo->isLittleEndian(),2129                        BC->AsmInfo->getCodePointerSize());2130  auto createReloc = [&](uint64_t Value, uint64_t Offset, uint64_t DwarfType) {2131    if (DwarfType == dwarf::DW_EH_PE_omit)2132      return;2133 2134    // Only fix references that are relative to other locations.2135    if (!(DwarfType & dwarf::DW_EH_PE_pcrel) &&2136        !(DwarfType & dwarf::DW_EH_PE_textrel) &&2137        !(DwarfType & dwarf::DW_EH_PE_funcrel) &&2138        !(DwarfType & dwarf::DW_EH_PE_datarel))2139      return;2140 2141    if (!(DwarfType & dwarf::DW_EH_PE_sdata4))2142      return;2143 2144    uint32_t RelType;2145    switch (DwarfType & 0x0f) {2146    default:2147      llvm_unreachable("unsupported DWARF encoding type");2148    case dwarf::DW_EH_PE_sdata4:2149    case dwarf::DW_EH_PE_udata4:2150      RelType = Relocation::getPC32();2151      Offset -= 4;2152      break;2153    case dwarf::DW_EH_PE_sdata8:2154    case dwarf::DW_EH_PE_udata8:2155      RelType = Relocation::getPC64();2156      Offset -= 8;2157      break;2158    }2159 2160    // Create a relocation against an absolute value since the goal is to2161    // preserve the contents of the section independent of the new values2162    // of referenced symbols.2163    RelocatedEHFrameSection->addRelocation(Offset, nullptr, RelType, Value);2164  };2165 2166  Error E = EHFrameParser::parse(DE, EHFrameSection->getAddress(), createReloc);2167  check_error(std::move(E), "failed to patch EH frame");2168}2169 2170Error RewriteInstance::readSpecialSections() {2171  NamedRegionTimer T("readSpecialSections", "read special sections",2172                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);2173 2174  bool HasTextRelocations = false;2175  bool HasSymbolTable = false;2176  bool HasDebugInfo = false;2177 2178  // Process special sections.2179  for (const SectionRef &Section : InputFile->sections()) {2180    Expected<StringRef> SectionNameOrErr = Section.getName();2181    check_error(SectionNameOrErr.takeError(), "cannot get section name");2182    StringRef SectionName = *SectionNameOrErr;2183 2184    if (Error E = Section.getContents().takeError())2185      return E;2186    BC->registerSection(Section);2187    LLVM_DEBUG(2188        dbgs() << "BOLT-DEBUG: registering section " << SectionName << " @ 0x"2189               << Twine::utohexstr(Section.getAddress()) << ":0x"2190               << Twine::utohexstr(Section.getAddress() + Section.getSize())2191               << "\n");2192    if (isDebugSection(SectionName))2193      HasDebugInfo = true;2194  }2195 2196  // Set IsRelro section attribute based on PT_GNU_RELRO segment.2197  markGnuRelroSections();2198 2199  if (HasDebugInfo && !opts::UpdateDebugSections && !opts::AggregateOnly) {2200    BC->errs() << "BOLT-WARNING: debug info will be stripped from the binary. "2201                  "Use -update-debug-sections to keep it.\n";2202  }2203 2204  HasTextRelocations = (bool)BC->getUniqueSectionByName(2205      ".rela" + std::string(BC->getMainCodeSectionName()));2206  HasSymbolTable = (bool)BC->getUniqueSectionByName(".symtab");2207  EHFrameSection = BC->getUniqueSectionByName(".eh_frame");2208 2209  if (ErrorOr<BinarySection &> BATSec =2210          BC->getUniqueSectionByName(BoltAddressTranslation::SECTION_NAME)) {2211    BC->HasBATSection = true;2212    // Do not read BAT when plotting a heatmap2213    if (opts::HeatmapMode != opts::HeatmapModeKind::HM_Exclusive) {2214      if (std::error_code EC = BAT->parse(BC->outs(), BATSec->getContents())) {2215        BC->errs() << "BOLT-ERROR: failed to parse BOLT address translation "2216                      "table.\n";2217        exit(1);2218      }2219    }2220  }2221 2222  if (opts::PrintSections) {2223    BC->outs() << "BOLT-INFO: Sections from original binary:\n";2224    BC->printSections(BC->outs());2225  }2226 2227  if (opts::RelocationMode == cl::BOU_TRUE && !HasTextRelocations) {2228    BC->errs()2229        << "BOLT-ERROR: relocations against code are missing from the input "2230           "file. Cannot proceed in relocations mode (-relocs).\n";2231    exit(1);2232  }2233 2234  BC->HasRelocations =2235      HasTextRelocations && (opts::RelocationMode != cl::BOU_FALSE);2236 2237  if (BC->IsLinuxKernel && BC->HasRelocations) {2238    BC->outs() << "BOLT-INFO: disabling relocation mode for Linux kernel\n";2239    BC->HasRelocations = false;2240  }2241 2242  BC->IsStripped = !HasSymbolTable;2243 2244  if (BC->IsStripped && !opts::AllowStripped) {2245    BC->errs()2246        << "BOLT-ERROR: stripped binaries are not supported. If you know "2247           "what you're doing, use --allow-stripped to proceed\n";2248    exit(1);2249  }2250 2251  // Force non-relocation mode for heatmap generation2252  if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive)2253    BC->HasRelocations = false;2254 2255  if (BC->HasRelocations)2256    BC->outs() << "BOLT-INFO: enabling " << (opts::StrictMode ? "strict " : "")2257               << "relocation mode\n";2258 2259  // Read EH frame for function boundaries info.2260  Expected<const DWARFDebugFrame *> EHFrameOrError = BC->DwCtx->getEHFrame();2261  if (!EHFrameOrError)2262    report_error("expected valid eh_frame section", EHFrameOrError.takeError());2263  CFIRdWrt.reset(new CFIReaderWriter(*BC, *EHFrameOrError.get()));2264 2265  processSectionMetadata();2266 2267  // Read .dynamic/PT_DYNAMIC.2268  return readELFDynamic();2269}2270 2271void RewriteInstance::adjustCommandLineOptions() {2272  if (BC->isAArch64() && !BC->HasRelocations)2273    BC->errs() << "BOLT-WARNING: non-relocation mode for AArch64 is not fully "2274                  "supported\n";2275 2276  if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary())2277    RtLibrary->adjustCommandLineOptions(*BC);2278 2279  if (BC->isX86() && BC->MAB->allowAutoPadding()) {2280    if (!BC->HasRelocations) {2281      BC->errs()2282          << "BOLT-ERROR: cannot apply mitigations for Intel JCC erratum in "2283             "non-relocation mode\n";2284      exit(1);2285    }2286    BC->outs()2287        << "BOLT-WARNING: using mitigation for Intel JCC erratum, layout "2288           "may take several minutes\n";2289  }2290 2291  if (opts::SplitEH && !BC->HasRelocations) {2292    BC->errs() << "BOLT-WARNING: disabling -split-eh in non-relocation mode\n";2293    opts::SplitEH = false;2294  }2295 2296  if (BC->isAArch64() && !opts::CompactCodeModel &&2297      opts::SplitStrategy == opts::SplitFunctionsStrategy::CDSplit) {2298    BC->errs() << "BOLT-ERROR: CDSplit is not supported with LongJmp. Try with "2299                  "'--compact-code-model'\n";2300    exit(1);2301  }2302 2303  if (opts::StrictMode && !BC->HasRelocations) {2304    BC->errs()2305        << "BOLT-WARNING: disabling strict mode (-strict) in non-relocation "2306           "mode\n";2307    opts::StrictMode = false;2308  }2309 2310  if (BC->HasRelocations && opts::AggregateOnly &&2311      !opts::StrictMode.getNumOccurrences()) {2312    BC->outs() << "BOLT-INFO: enabling strict relocation mode for aggregation "2313                  "purposes\n";2314    opts::StrictMode = true;2315  }2316 2317  if (!BC->HasRelocations &&2318      opts::ReorderFunctions != ReorderFunctions::RT_NONE) {2319    BC->errs() << "BOLT-ERROR: function reordering only works when "2320               << "relocations are enabled\n";2321    exit(1);2322  }2323 2324  if (!BC->HasRelocations &&2325      opts::ICF == IdenticalCodeFolding::ICFLevel::Safe) {2326    BC->errs() << "BOLT-ERROR: binary built without relocations. Safe ICF is "2327                  "not supported\n";2328    exit(1);2329  }2330 2331  if (opts::Instrument ||2332      (opts::ReorderFunctions != ReorderFunctions::RT_NONE &&2333       !opts::HotText.getNumOccurrences())) {2334    opts::HotText = true;2335  } else if (opts::HotText && !BC->HasRelocations) {2336    BC->errs() << "BOLT-WARNING: hot text is disabled in non-relocation mode\n";2337    opts::HotText = false;2338  }2339 2340  if (opts::Instrument && opts::UseGnuStack) {2341    BC->errs() << "BOLT-ERROR: cannot avoid having writeable and executable "2342                  "segment in instrumented binary if program headers will be "2343                  "updated in place\n";2344    exit(1);2345  }2346 2347  if (opts::Instrument && opts::RuntimeLibInitHook == opts::RLIH_ENTRY_POINT &&2348      !BC->HasInterpHeader) {2349    BC->errs()2350        << "BOLT-WARNING: adjusted runtime-lib-init-hook to 'init' due to "2351           "absence of INTERP header\n";2352    opts::RuntimeLibInitHook = opts::RLIH_INIT;2353  }2354 2355  if (opts::HotText && opts::HotTextMoveSections.getNumOccurrences() == 0) {2356    opts::HotTextMoveSections.addValue(".stub");2357    opts::HotTextMoveSections.addValue(".mover");2358    opts::HotTextMoveSections.addValue(".never_hugify");2359  }2360 2361  if (opts::UseOldText && !BC->OldTextSectionAddress) {2362    BC->errs()2363        << "BOLT-WARNING: cannot use old .text as the section was not found"2364           "\n";2365    opts::UseOldText = false;2366  }2367  if (opts::UseOldText && !BC->HasRelocations) {2368    BC->errs() << "BOLT-WARNING: cannot use old .text in non-relocation mode\n";2369    opts::UseOldText = false;2370  }2371 2372  if (!opts::AlignText.getNumOccurrences())2373    opts::AlignText = BC->PageAlign;2374 2375  if (opts::AlignText < opts::AlignFunctions)2376    opts::AlignText = (unsigned)opts::AlignFunctions;2377 2378  if (BC->isX86() && opts::Lite.getNumOccurrences() == 0 && !opts::StrictMode &&2379      !opts::UseOldText)2380    opts::Lite = true;2381 2382  if (opts::Lite && opts::UseOldText) {2383    BC->errs() << "BOLT-WARNING: cannot combine -lite with -use-old-text. "2384                  "Disabling -use-old-text.\n";2385    opts::UseOldText = false;2386  }2387 2388  if (opts::Lite && opts::StrictMode) {2389    BC->errs()2390        << "BOLT-ERROR: -strict and -lite cannot be used at the same time\n";2391    exit(1);2392  }2393 2394  if (opts::Lite)2395    BC->outs() << "BOLT-INFO: enabling lite mode\n";2396 2397  if (BC->IsLinuxKernel) {2398    if (!opts::KeepNops.getNumOccurrences())2399      opts::KeepNops = true;2400 2401    // Linux kernel may resume execution after a trap or x86 HLT instruction.2402    if (!opts::TerminalHLT.getNumOccurrences())2403      opts::TerminalHLT = false;2404    if (!opts::TerminalTrap.getNumOccurrences())2405      opts::TerminalTrap = false;2406  }2407}2408 2409namespace {2410template <typename ELFT>2411int64_t getRelocationAddend(const ELFObjectFile<ELFT> *Obj,2412                            const RelocationRef &RelRef) {2413  using ELFShdrTy = typename ELFT::Shdr;2414  using Elf_Rela = typename ELFT::Rela;2415  int64_t Addend = 0;2416  const ELFFile<ELFT> &EF = Obj->getELFFile();2417  DataRefImpl Rel = RelRef.getRawDataRefImpl();2418  const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a));2419  switch (RelocationSection->sh_type) {2420  default:2421    llvm_unreachable("unexpected relocation section type");2422  case ELF::SHT_REL:2423    break;2424  case ELF::SHT_RELA: {2425    const Elf_Rela *RelA = Obj->getRela(Rel);2426    Addend = RelA->r_addend;2427    break;2428  }2429  }2430 2431  return Addend;2432}2433 2434int64_t getRelocationAddend(const ELFObjectFileBase *Obj,2435                            const RelocationRef &Rel) {2436  return getRelocationAddend(cast<ELF64LEObjectFile>(Obj), Rel);2437}2438 2439template <typename ELFT>2440uint32_t getRelocationSymbol(const ELFObjectFile<ELFT> *Obj,2441                             const RelocationRef &RelRef) {2442  using ELFShdrTy = typename ELFT::Shdr;2443  uint32_t Symbol = 0;2444  const ELFFile<ELFT> &EF = Obj->getELFFile();2445  DataRefImpl Rel = RelRef.getRawDataRefImpl();2446  const ELFShdrTy *RelocationSection = cantFail(EF.getSection(Rel.d.a));2447  switch (RelocationSection->sh_type) {2448  default:2449    llvm_unreachable("unexpected relocation section type");2450  case ELF::SHT_REL:2451    Symbol = Obj->getRel(Rel)->getSymbol(EF.isMips64EL());2452    break;2453  case ELF::SHT_RELA:2454    Symbol = Obj->getRela(Rel)->getSymbol(EF.isMips64EL());2455    break;2456  }2457 2458  return Symbol;2459}2460 2461uint32_t getRelocationSymbol(const ELFObjectFileBase *Obj,2462                             const RelocationRef &Rel) {2463  return getRelocationSymbol(cast<ELF64LEObjectFile>(Obj), Rel);2464}2465} // anonymous namespace2466 2467bool RewriteInstance::analyzeRelocation(2468    const RelocationRef &Rel, uint32_t &RType, std::string &SymbolName,2469    bool &IsSectionRelocation, uint64_t &SymbolAddress, int64_t &Addend,2470    uint64_t &ExtractedValue) const {2471  if (!Relocation::isSupported(RType))2472    return false;2473 2474  auto IsWeakReference = [](const SymbolRef &Symbol) {2475    Expected<uint32_t> SymFlagsOrErr = Symbol.getFlags();2476    if (!SymFlagsOrErr)2477      return false;2478    return (*SymFlagsOrErr & SymbolRef::SF_Undefined) &&2479           (*SymFlagsOrErr & SymbolRef::SF_Weak);2480  };2481 2482  const bool IsAArch64 = BC->isAArch64();2483 2484  const size_t RelSize = Relocation::getSizeForType(RType);2485 2486  ErrorOr<uint64_t> Value =2487      BC->getUnsignedValueAtAddress(Rel.getOffset(), RelSize);2488  assert(Value && "failed to extract relocated value");2489 2490  ExtractedValue = Relocation::extractValue(RType, *Value, Rel.getOffset());2491  Addend = getRelocationAddend(InputFile, Rel);2492 2493  const bool IsPCRelative = Relocation::isPCRelative(RType);2494  const uint64_t PCRelOffset = IsPCRelative && !IsAArch64 ? Rel.getOffset() : 0;2495  bool SkipVerification = false;2496  auto SymbolIter = Rel.getSymbol();2497  if (SymbolIter == InputFile->symbol_end()) {2498    SymbolAddress = ExtractedValue - Addend + PCRelOffset;2499    MCSymbol *RelSymbol =2500        BC->getOrCreateGlobalSymbol(SymbolAddress, "RELSYMat");2501    SymbolName = std::string(RelSymbol->getName());2502    IsSectionRelocation = false;2503  } else {2504    const SymbolRef &Symbol = *SymbolIter;2505    SymbolName = std::string(cantFail(Symbol.getName()));2506    SymbolAddress = cantFail(Symbol.getAddress());2507    SkipVerification = (cantFail(Symbol.getType()) == SymbolRef::ST_Other);2508    // Section symbols are marked as ST_Debug.2509    IsSectionRelocation = (cantFail(Symbol.getType()) == SymbolRef::ST_Debug);2510    // Check for PLT entry registered with symbol name2511    if (!SymbolAddress && !IsWeakReference(Symbol) &&2512        (IsAArch64 || BC->isRISCV())) {2513      const BinaryData *BD = BC->getPLTBinaryDataByName(SymbolName);2514      SymbolAddress = BD ? BD->getAddress() : 0;2515    }2516  }2517  // For PIE or dynamic libs, the linker may choose not to put the relocation2518  // result at the address if it is a X86_64_64 one because it will emit a2519  // dynamic relocation (X86_RELATIVE) for the dynamic linker and loader to2520  // resolve it at run time. The static relocation result goes as the addend2521  // of the dynamic relocation in this case. We can't verify these cases.2522  // FIXME: perhaps we can try to find if it really emitted a corresponding2523  // RELATIVE relocation at this offset with the correct value as the addend.2524  if (!BC->HasFixedLoadAddress && RelSize == 8)2525    SkipVerification = true;2526 2527  if (IsSectionRelocation && !IsAArch64) {2528    ErrorOr<BinarySection &> Section = BC->getSectionForAddress(SymbolAddress);2529    assert(Section && "section expected for section relocation");2530    SymbolName = "section " + std::string(Section->getName());2531    // Convert section symbol relocations to regular relocations inside2532    // non-section symbols.2533    if (Section->containsAddress(ExtractedValue) && !IsPCRelative) {2534      SymbolAddress = ExtractedValue;2535      Addend = 0;2536    } else {2537      Addend = ExtractedValue - (SymbolAddress - PCRelOffset);2538    }2539  }2540 2541  // GOT relocation can cause the underlying instruction to be modified by the2542  // linker, resulting in the extracted value being different from the actual2543  // symbol. It's also possible to have a GOT entry for a symbol defined in the2544  // binary. In the latter case, the instruction can be using the GOT version2545  // causing the extracted value mismatch. Similar cases can happen for TLS.2546  // Pass the relocation information as is to the disassembler and let it decide2547  // how to use it for the operand symbolization.2548  if (Relocation::isGOT(RType) || Relocation::isTLS(RType)) {2549    SkipVerification = true;2550  } else if (!SymbolAddress) {2551    assert(!IsSectionRelocation);2552    if (ExtractedValue || Addend == 0 || IsPCRelative) {2553      SymbolAddress =2554          truncateToSize(ExtractedValue - Addend + PCRelOffset, RelSize);2555    } else {2556      // This is weird case.  The extracted value is zero but the addend is2557      // non-zero and the relocation is not pc-rel.  Using the previous logic,2558      // the SymbolAddress would end up as a huge number.  Seen in2559      // exceptions_pic.test.2560      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: relocation @ 0x"2561                        << Twine::utohexstr(Rel.getOffset())2562                        << " value does not match addend for "2563                        << "relocation to undefined symbol.\n");2564      return true;2565    }2566  }2567 2568  auto verifyExtractedValue = [&]() {2569    if (SkipVerification)2570      return true;2571 2572    if (IsAArch64 || BC->isRISCV())2573      return true;2574 2575    if (SymbolName == "__hot_start" || SymbolName == "__hot_end")2576      return true;2577 2578    if (RType == ELF::R_X86_64_PLT32)2579      return true;2580 2581    return truncateToSize(ExtractedValue, RelSize) ==2582           truncateToSize(SymbolAddress + Addend - PCRelOffset, RelSize);2583  };2584 2585  (void)verifyExtractedValue;2586  assert(verifyExtractedValue() && "mismatched extracted relocation value");2587 2588  return true;2589}2590 2591void RewriteInstance::processDynamicRelocations() {2592  // Read .relr.dyn section containing compressed R_*_RELATIVE relocations.2593  if (DynamicRelrSize > 0) {2594    ErrorOr<BinarySection &> DynamicRelrSectionOrErr =2595        BC->getSectionForAddress(*DynamicRelrAddress);2596    if (!DynamicRelrSectionOrErr)2597      report_error("unable to find section corresponding to DT_RELR",2598                   DynamicRelrSectionOrErr.getError());2599    if (DynamicRelrSectionOrErr->getSize() != DynamicRelrSize)2600      report_error("section size mismatch for DT_RELRSZ",2601                   errc::executable_format_error);2602    readDynamicRelrRelocations(*DynamicRelrSectionOrErr);2603  }2604 2605  // Read relocations for PLT - DT_JMPREL.2606  if (PLTRelocationsSize > 0) {2607    ErrorOr<BinarySection &> PLTRelSectionOrErr =2608        BC->getSectionForAddress(*PLTRelocationsAddress);2609    if (!PLTRelSectionOrErr)2610      report_error("unable to find section corresponding to DT_JMPREL",2611                   PLTRelSectionOrErr.getError());2612    if (PLTRelSectionOrErr->getSize() != PLTRelocationsSize)2613      report_error("section size mismatch for DT_PLTRELSZ",2614                   errc::executable_format_error);2615    readDynamicRelocations(PLTRelSectionOrErr->getSectionRef(),2616                           /*IsJmpRel*/ true);2617  }2618 2619  // The rest of dynamic relocations - DT_RELA.2620  // The static executable might have .rela.dyn section and not have PT_DYNAMIC2621  if (!DynamicRelocationsSize && BC->IsStaticExecutable) {2622    ErrorOr<BinarySection &> DynamicRelSectionOrErr =2623        BC->getUniqueSectionByName(getRelaDynSectionName());2624    if (DynamicRelSectionOrErr) {2625      DynamicRelocationsAddress = DynamicRelSectionOrErr->getAddress();2626      DynamicRelocationsSize = DynamicRelSectionOrErr->getSize();2627      const SectionRef &SectionRef = DynamicRelSectionOrErr->getSectionRef();2628      DynamicRelativeRelocationsCount = std::distance(2629          SectionRef.relocation_begin(), SectionRef.relocation_end());2630    }2631  }2632 2633  if (DynamicRelocationsSize > 0) {2634    ErrorOr<BinarySection &> DynamicRelSectionOrErr =2635        BC->getSectionForAddress(*DynamicRelocationsAddress);2636    if (!DynamicRelSectionOrErr)2637      report_error("unable to find section corresponding to DT_RELA",2638                   DynamicRelSectionOrErr.getError());2639    auto DynamicRelSectionSize = DynamicRelSectionOrErr->getSize();2640    // On RISC-V DT_RELASZ seems to include both .rela.dyn and .rela.plt2641    if (DynamicRelocationsSize == DynamicRelSectionSize + PLTRelocationsSize)2642      DynamicRelocationsSize = DynamicRelSectionSize;2643    if (DynamicRelSectionSize != DynamicRelocationsSize)2644      report_error("section size mismatch for DT_RELASZ",2645                   errc::executable_format_error);2646    readDynamicRelocations(DynamicRelSectionOrErr->getSectionRef(),2647                           /*IsJmpRel*/ false);2648  }2649}2650 2651void RewriteInstance::processRelocations() {2652  if (!BC->HasRelocations)2653    return;2654 2655  for (const SectionRef &Section : InputFile->sections()) {2656    section_iterator SecIter = cantFail(Section.getRelocatedSection());2657    if (SecIter == InputFile->section_end())2658      continue;2659    if (BinarySection(*BC, Section).isAllocatable())2660      continue;2661 2662    readRelocations(Section);2663  }2664 2665  if (NumFailedRelocations)2666    BC->errs() << "BOLT-WARNING: Failed to analyze " << NumFailedRelocations2667               << " relocations\n";2668}2669 2670void RewriteInstance::readDynamicRelocations(const SectionRef &Section,2671                                             bool IsJmpRel) {2672  assert(BinarySection(*BC, Section).isAllocatable() && "allocatable expected");2673 2674  LLVM_DEBUG({2675    StringRef SectionName = cantFail(Section.getName());2676    dbgs() << "BOLT-DEBUG: reading relocations for section " << SectionName2677           << ":\n";2678  });2679 2680  for (const RelocationRef &Rel : Section.relocations()) {2681    const uint32_t RType = Relocation::getType(Rel);2682    if (Relocation::isNone(RType))2683      continue;2684 2685    StringRef SymbolName = "<none>";2686    MCSymbol *Symbol = nullptr;2687    uint64_t SymbolAddress = 0;2688    const uint64_t Addend = getRelocationAddend(InputFile, Rel);2689 2690    symbol_iterator SymbolIter = Rel.getSymbol();2691    if (SymbolIter != InputFile->symbol_end()) {2692      SymbolName = cantFail(SymbolIter->getName());2693      BinaryData *BD = BC->getBinaryDataByName(SymbolName);2694      Symbol = BD ? BD->getSymbol()2695                  : BC->getOrCreateUndefinedGlobalSymbol(SymbolName);2696      SymbolAddress = cantFail(SymbolIter->getAddress());2697      (void)SymbolAddress;2698    }2699 2700    LLVM_DEBUG(2701      SmallString<16> TypeName;2702      Rel.getTypeName(TypeName);2703      dbgs() << "BOLT-DEBUG: dynamic relocation at 0x"2704             << Twine::utohexstr(Rel.getOffset()) << " : " << TypeName2705             << " : " << SymbolName << " : " <<  Twine::utohexstr(SymbolAddress)2706             << " : + 0x" << Twine::utohexstr(Addend) << '\n'2707    );2708 2709    if (IsJmpRel)2710      IsJmpRelocation[RType] = true;2711 2712    if (Symbol)2713      SymbolIndex[Symbol] = getRelocationSymbol(InputFile, Rel);2714 2715    const uint64_t ReferencedAddress = SymbolAddress + Addend;2716    BinaryFunction *Func =2717        BC->getBinaryFunctionContainingAddress(ReferencedAddress);2718 2719    if (Relocation::isRelative(RType) && SymbolAddress == 0) {2720      if (Func) {2721        if (!Func->isInConstantIsland(ReferencedAddress)) {2722          if (const uint64_t ReferenceOffset =2723                  ReferencedAddress - Func->getAddress()) {2724            Func->addEntryPointAtOffset(ReferenceOffset);2725          }2726        } else {2727          BC->errs() << "BOLT-ERROR: referenced address at 0x"2728                     << Twine::utohexstr(ReferencedAddress)2729                     << " is in constant island of function " << *Func << "\n";2730          exit(1);2731        }2732      }2733    } else if (Relocation::isRelative(RType) && SymbolAddress != 0) {2734      BC->errs() << "BOLT-ERROR: symbol address non zero for RELATIVE "2735                    "relocation type\n";2736      exit(1);2737    }2738 2739    BC->addDynamicRelocation(Rel.getOffset(), Symbol, RType, Addend);2740  }2741}2742 2743void RewriteInstance::readDynamicRelrRelocations(BinarySection &Section) {2744  assert(Section.isAllocatable() && "allocatable expected");2745 2746  LLVM_DEBUG({2747    StringRef SectionName = Section.getName();2748    dbgs() << "BOLT-DEBUG: reading relocations in section " << SectionName2749           << ":\n";2750  });2751 2752  const uint32_t RType = Relocation::getRelative();2753  const uint8_t PSize = BC->AsmInfo->getCodePointerSize();2754  const uint64_t MaxDelta = ((CHAR_BIT * DynamicRelrEntrySize) - 1) * PSize;2755 2756  auto ExtractAddendValue = [&](uint64_t Address) -> uint64_t {2757    ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);2758    assert(Section && "cannot get section for data address from RELR");2759    DataExtractor DE = DataExtractor(Section->getContents(),2760                                     BC->AsmInfo->isLittleEndian(), PSize);2761    uint64_t Offset = Address - Section->getAddress();2762    return DE.getUnsigned(&Offset, PSize);2763  };2764 2765  auto AddRelocation = [&](uint64_t Address) {2766    uint64_t Addend = ExtractAddendValue(Address);2767    LLVM_DEBUG(dbgs() << "BOLT-DEBUG: R_*_RELATIVE relocation at 0x"2768                      << Twine::utohexstr(Address) << " to 0x"2769                      << Twine::utohexstr(Addend) << '\n';);2770    BC->addDynamicRelocation(Address, nullptr, RType, Addend);2771  };2772 2773  DataExtractor DE = DataExtractor(Section.getContents(),2774                                   BC->AsmInfo->isLittleEndian(), PSize);2775  uint64_t Offset = 0, Address = 0;2776  uint64_t RelrCount = DynamicRelrSize / DynamicRelrEntrySize;2777  while (RelrCount--) {2778    assert(DE.isValidOffset(Offset));2779    uint64_t Entry = DE.getUnsigned(&Offset, DynamicRelrEntrySize);2780    if ((Entry & 1) == 0) {2781      AddRelocation(Entry);2782      Address = Entry + PSize;2783    } else {2784      const uint64_t StartAddress = Address;2785      while (Entry >>= 1) {2786        if (Entry & 1)2787          AddRelocation(Address);2788 2789        Address += PSize;2790      }2791 2792      Address = StartAddress + MaxDelta;2793    }2794  }2795}2796 2797void RewriteInstance::printRelocationInfo(const RelocationRef &Rel,2798                                          StringRef SymbolName,2799                                          uint64_t SymbolAddress,2800                                          uint64_t Addend,2801                                          uint64_t ExtractedValue) const {2802  SmallString<16> TypeName;2803  Rel.getTypeName(TypeName);2804  const uint64_t Address = SymbolAddress + Addend;2805  const uint64_t Offset = Rel.getOffset();2806  ErrorOr<BinarySection &> Section = BC->getSectionForAddress(SymbolAddress);2807  BinaryFunction *Func =2808      BC->getBinaryFunctionContainingAddress(Offset, false, BC->isAArch64());2809  dbgs() << formatv("Relocation: offset = {0:x}; type = {1}; value = {2:x}; ",2810                    Offset, TypeName, ExtractedValue)2811         << formatv("symbol = {0} ({1}); symbol address = {2:x}; ", SymbolName,2812                    Section ? Section->getName() : "", SymbolAddress)2813         << formatv("addend = {0:x}; address = {1:x}; in = ", Addend, Address);2814  if (Func)2815    dbgs() << Func->getPrintName();2816  else2817    dbgs() << BC->getSectionForAddress(Rel.getOffset())->getName();2818  dbgs() << '\n';2819}2820 2821void RewriteInstance::readRelocations(const SectionRef &Section) {2822  LLVM_DEBUG({2823    StringRef SectionName = cantFail(Section.getName());2824    dbgs() << "BOLT-DEBUG: reading relocations for section " << SectionName2825           << ":\n";2826  });2827  if (BinarySection(*BC, Section).isAllocatable()) {2828    LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring runtime relocations\n");2829    return;2830  }2831  section_iterator SecIter = cantFail(Section.getRelocatedSection());2832  assert(SecIter != InputFile->section_end() && "relocated section expected");2833  SectionRef RelocatedSection = *SecIter;2834 2835  StringRef RelocatedSectionName = cantFail(RelocatedSection.getName());2836  LLVM_DEBUG(dbgs() << "BOLT-DEBUG: relocated section is "2837                    << RelocatedSectionName << '\n');2838 2839  if (!BinarySection(*BC, RelocatedSection).isAllocatable()) {2840    LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocations against "2841                      << "non-allocatable section\n");2842    return;2843  }2844  const bool SkipRelocs = StringSwitch<bool>(RelocatedSectionName)2845                              .Cases({".plt", ".rela.plt", ".got.plt",2846                                      ".eh_frame", ".gcc_except_table"},2847                                     true)2848                              .Default(false);2849  if (SkipRelocs) {2850    LLVM_DEBUG(2851        dbgs() << "BOLT-DEBUG: ignoring relocations against known section\n");2852    return;2853  }2854 2855  for (const RelocationRef &Rel : Section.relocations())2856    handleRelocation(RelocatedSection, Rel);2857}2858 2859void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,2860                                       const RelocationRef &Rel) {2861  const bool IsAArch64 = BC->isAArch64();2862  const bool IsX86 = BC->isX86();2863  const bool IsFromCode = RelocatedSection.isText();2864  const bool IsWritable = BinarySection(*BC, RelocatedSection).isWritable();2865 2866  SmallString<16> TypeName;2867  Rel.getTypeName(TypeName);2868  uint32_t RType = Relocation::getType(Rel);2869  if (Relocation::skipRelocationType(RType))2870    return;2871 2872  // Adjust the relocation type as the linker might have skewed it.2873  if (IsX86 && (RType & ELF::R_X86_64_converted_reloc_bit)) {2874    if (opts::Verbosity >= 1)2875      dbgs() << "BOLT-WARNING: ignoring R_X86_64_converted_reloc_bit\n";2876    RType &= ~ELF::R_X86_64_converted_reloc_bit;2877  }2878 2879  if (Relocation::isTLS(RType)) {2880    // No special handling required for TLS relocations on X86.2881    if (IsX86)2882      return;2883 2884    // The non-got related TLS relocations on AArch64 and RISC-V also could be2885    // skipped.2886    if (!Relocation::isGOT(RType))2887      return;2888  }2889 2890  if (!IsAArch64 && BC->getDynamicRelocationAt(Rel.getOffset())) {2891    LLVM_DEBUG({2892      dbgs() << formatv("BOLT-DEBUG: address {0:x} has a ", Rel.getOffset())2893             << "dynamic relocation against it. Ignoring static relocation.\n";2894    });2895    return;2896  }2897 2898  std::string SymbolName;2899  uint64_t SymbolAddress;2900  int64_t Addend;2901  uint64_t ExtractedValue;2902  bool IsSectionRelocation;2903  if (!analyzeRelocation(Rel, RType, SymbolName, IsSectionRelocation,2904                         SymbolAddress, Addend, ExtractedValue)) {2905    LLVM_DEBUG({2906      dbgs() << "BOLT-WARNING: failed to analyze relocation @ offset = "2907             << formatv("{0:x}; type name = {1}\n", Rel.getOffset(), TypeName);2908    });2909    ++NumFailedRelocations;2910    return;2911  }2912 2913  if (!IsFromCode && !IsWritable && (IsX86 || IsAArch64) &&2914      Relocation::isPCRelative(RType)) {2915    BinaryData *BD = BC->getBinaryDataContainingAddress(Rel.getOffset());2916    if (BD && (BD->nameStartsWith("_ZTV") ||   // vtable2917               BD->nameStartsWith("_ZTCN"))) { // construction vtable2918      BinaryFunction *BF = BC->getBinaryFunctionContainingAddress(2919          SymbolAddress, /*CheckPastEnd*/ false, /*UseMaxSize*/ true);2920      if (BF) {2921        if (BF->getAddress() != SymbolAddress) {2922          BC->errs()2923              << "BOLT-ERROR: the virtual function table entry at offset 0x"2924              << Twine::utohexstr(Rel.getOffset())2925              << " points to the middle of a function @ 0x"2926              << Twine::utohexstr(BF->getAddress()) << "\n";2927          exit(1);2928        }2929        BC->addRelocation(Rel.getOffset(), BF->getSymbol(), RType, Addend,2930                          ExtractedValue);2931        return;2932      }2933    }2934  }2935 2936  const uint64_t Address = SymbolAddress + Addend;2937 2938  LLVM_DEBUG({2939    dbgs() << "BOLT-DEBUG: ";2940    printRelocationInfo(Rel, SymbolName, SymbolAddress, Addend, ExtractedValue);2941  });2942 2943  BinaryFunction *ContainingBF = nullptr;2944  if (IsFromCode) {2945    ContainingBF =2946        BC->getBinaryFunctionContainingAddress(Rel.getOffset(),2947                                               /*CheckPastEnd*/ false,2948                                               /*UseMaxSize*/ true);2949    assert(ContainingBF && "cannot find function for address in code");2950    if (!IsAArch64 && !ContainingBF->containsAddress(Rel.getOffset())) {2951      if (opts::Verbosity >= 1)2952        BC->outs() << formatv(2953            "BOLT-INFO: {0} has relocations in padding area\n", *ContainingBF);2954      ContainingBF->setSize(ContainingBF->getMaxSize());2955      ContainingBF->setSimple(false);2956      return;2957    }2958  }2959 2960  MCSymbol *ReferencedSymbol = nullptr;2961  if (!IsSectionRelocation) {2962    if (BinaryData *BD = BC->getBinaryDataByName(SymbolName)) {2963      ReferencedSymbol = BD->getSymbol();2964    } else if (BC->isGOTSymbol(SymbolName)) {2965      if (BinaryData *BD = BC->getGOTSymbol())2966        ReferencedSymbol = BD->getSymbol();2967    } else if (BinaryData *BD = BC->getBinaryDataAtAddress(SymbolAddress)) {2968      ReferencedSymbol = BD->getSymbol();2969    }2970  }2971 2972  ErrorOr<BinarySection &> ReferencedSection{std::errc::bad_address};2973  symbol_iterator SymbolIter = Rel.getSymbol();2974  if (SymbolIter != InputFile->symbol_end()) {2975    SymbolRef Symbol = *SymbolIter;2976    section_iterator Section =2977        cantFail(Symbol.getSection(), "cannot get symbol section");2978    if (Section != InputFile->section_end()) {2979      Expected<StringRef> SectionName = Section->getName();2980      if (SectionName && !SectionName->empty())2981        ReferencedSection = BC->getUniqueSectionByName(*SectionName);2982    } else if (BC->isRISCV() && ReferencedSymbol && ContainingBF &&2983               (cantFail(Symbol.getFlags()) & SymbolRef::SF_Absolute)) {2984      // This might be a relocation for an ABS symbols like __global_pointer$ on2985      // RISC-V2986      ContainingBF->addRelocation(Rel.getOffset(), ReferencedSymbol,2987                                  Relocation::getType(Rel), 0,2988                                  cantFail(Symbol.getValue()));2989      return;2990    }2991  }2992 2993  if (!ReferencedSection)2994    ReferencedSection = BC->getSectionForAddress(SymbolAddress);2995 2996  const bool IsToCode = ReferencedSection && ReferencedSection->isText();2997 2998  // Special handling of PC-relative relocations.2999  if (IsX86 && Relocation::isPCRelative(RType)) {3000    if (!IsFromCode && IsToCode) {3001      // PC-relative relocations from data to code are tricky since the3002      // original information is typically lost after linking, even with3003      // '--emit-relocs'. Such relocations are normally used by PIC-style3004      // jump tables and they reference both the jump table and jump3005      // targets by computing the difference between the two. If we blindly3006      // apply the relocation, it will appear that it references an arbitrary3007      // location in the code, possibly in a different function from the one3008      // containing the jump table.3009      //3010      // For that reason, we only register the fact that there is a3011      // PC-relative relocation at a given address against the code.3012      // The actual referenced label/address will be determined during jump3013      // table analysis.3014      BC->addPCRelativeDataRelocation(Rel.getOffset());3015    } else if (ContainingBF && !IsSectionRelocation && ReferencedSymbol) {3016      // If we know the referenced symbol, register the relocation from3017      // the code. It's required  to properly handle cases where3018      // "symbol + addend" references an object different from "symbol".3019      ContainingBF->addRelocation(Rel.getOffset(), ReferencedSymbol, RType,3020                                  Addend, ExtractedValue);3021    } else {3022      LLVM_DEBUG({3023        dbgs() << "BOLT-DEBUG: not creating PC-relative relocation at"3024               << formatv("{0:x} for {1}\n", Rel.getOffset(), SymbolName);3025      });3026    }3027 3028    return;3029  }3030 3031  bool ForceRelocation = BC->forceSymbolRelocations(SymbolName);3032  if ((BC->isAArch64() || BC->isRISCV()) && Relocation::isGOT(RType))3033    ForceRelocation = true;3034 3035  if (!ReferencedSection && !ForceRelocation) {3036    LLVM_DEBUG(dbgs() << "BOLT-DEBUG: cannot determine referenced section.\n");3037    return;3038  }3039 3040  // Occasionally we may see a reference past the last byte of the function3041  // typically as a result of __builtin_unreachable(). Check it here.3042  BinaryFunction *ReferencedBF = BC->getBinaryFunctionContainingAddress(3043      Address, /*CheckPastEnd*/ true, /*UseMaxSize*/ IsAArch64);3044 3045  if (!IsSectionRelocation) {3046    if (BinaryFunction *BF =3047            BC->getBinaryFunctionContainingAddress(SymbolAddress)) {3048      if (BF != ReferencedBF) {3049        // It's possible we are referencing a function without referencing any3050        // code, e.g. when taking a bitmask action on a function address.3051        BC->errs()3052            << "BOLT-WARNING: non-standard function reference (e.g. bitmask)"3053            << formatv(" detected against function {0} from ", *BF);3054        if (IsFromCode)3055          BC->errs() << formatv("function {0}\n", *ContainingBF);3056        else3057          BC->errs() << formatv("data section at {0:x}\n", Rel.getOffset());3058        LLVM_DEBUG(printRelocationInfo(Rel, SymbolName, SymbolAddress, Addend,3059                                       ExtractedValue));3060        ReferencedBF = BF;3061      }3062    }3063  } else if (ReferencedBF) {3064    assert(ReferencedSection && "section expected for section relocation");3065    if (*ReferencedBF->getOriginSection() != *ReferencedSection) {3066      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring false function reference\n");3067      ReferencedBF = nullptr;3068    }3069  }3070 3071  // Workaround for a member function pointer de-virtualization bug. We check3072  // if a non-pc-relative relocation in the code is pointing to (fptr - 1).3073  if (IsToCode && ContainingBF && !Relocation::isPCRelative(RType) &&3074      (!ReferencedBF || (ReferencedBF->getAddress() != Address))) {3075    if (const BinaryFunction *RogueBF =3076            BC->getBinaryFunctionAtAddress(Address + 1)) {3077      // Do an extra check that the function was referenced previously.3078      // It's a linear search, but it should rarely happen.3079      auto CheckReloc = [&](const Relocation &Rel) {3080        return Rel.Symbol == RogueBF->getSymbol() &&3081               !Relocation::isPCRelative(Rel.Type);3082      };3083      bool Found = llvm::any_of(3084          llvm::make_second_range(ContainingBF->Relocations), CheckReloc);3085 3086      if (Found) {3087        BC->errs()3088            << "BOLT-WARNING: detected possible compiler de-virtualization "3089               "bug: -1 addend used with non-pc-relative relocation against "3090            << formatv("function {0} in function {1}\n", *RogueBF,3091                       *ContainingBF);3092        return;3093      }3094    }3095  }3096 3097  if (ForceRelocation && !ReferencedBF) {3098    // Create the relocation symbol if it's not defined in the binary.3099    if (SymbolAddress == 0)3100      ReferencedSymbol = BC->registerNameAtAddress(SymbolName, 0, 0, 0);3101 3102    LLVM_DEBUG(3103        dbgs() << "BOLT-DEBUG: forcing relocation against symbol "3104               << (ReferencedSymbol ? ReferencedSymbol->getName() : "<none>")3105               << " with addend " << Addend << '\n');3106  } else if (ReferencedBF) {3107    ReferencedSymbol = ReferencedBF->getSymbol();3108    uint64_t RefFunctionOffset = 0;3109 3110    // Adjust the point of reference to a code location inside a function.3111    if (ReferencedBF->containsAddress(Address, /*UseMaxSize = */ true)) {3112      RefFunctionOffset = Address - ReferencedBF->getAddress();3113      if (Relocation::isInstructionReference(RType)) {3114        // Instruction labels are created while disassembling so we just leave3115        // the symbol empty for now. Since the extracted value is typically3116        // unrelated to the referenced symbol (e.g., %pcrel_lo in RISC-V3117        // references an instruction but the patched value references the low3118        // bits of a data address), we set the extracted value to the symbol3119        // address in order to be able to correctly reconstruct the reference3120        // later.3121        ReferencedSymbol = nullptr;3122        ExtractedValue = Address;3123      } else if (RefFunctionOffset) {3124        if (ContainingBF && ContainingBF != ReferencedBF &&3125            !ReferencedBF->isInConstantIsland(Address)) {3126          ReferencedSymbol =3127              ReferencedBF->addEntryPointAtOffset(RefFunctionOffset);3128        } else {3129          ReferencedSymbol = ReferencedBF->getOrCreateLocalLabel(Address);3130 3131          // If ContainingBF != nullptr, it equals ReferencedBF (see3132          // if-condition above) so we're handling a relocation from a function3133          // to itself. RISC-V uses such relocations for branches, for example.3134          // These should not be registered as externally references offsets.3135          if (!ContainingBF && !ReferencedBF->isInConstantIsland(Address)) {3136            ReferencedBF->registerInternalRefDataRelocation(RefFunctionOffset,3137                                                            Rel.getOffset());3138          }3139        }3140        if (opts::Verbosity > 1 &&3141            BinarySection(*BC, RelocatedSection).isWritable())3142          BC->errs()3143              << "BOLT-WARNING: writable reference into the middle of the "3144              << formatv("function {0} detected at address {1:x}\n",3145                         *ReferencedBF, Rel.getOffset());3146      }3147      SymbolAddress = Address;3148      Addend = 0;3149    }3150    LLVM_DEBUG({3151      dbgs() << "  referenced function " << *ReferencedBF;3152      if (Address != ReferencedBF->getAddress())3153        dbgs() << formatv(" at offset {0:x}", RefFunctionOffset);3154      dbgs() << '\n';3155    });3156  } else {3157    if (IsToCode && SymbolAddress) {3158      // This can happen e.g. with PIC-style jump tables.3159      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: no corresponding function for "3160                           "relocation against code\n");3161    }3162 3163    // In AArch64 there are zero reasons to keep a reference to the3164    // "original" symbol plus addend. The original symbol is probably just a3165    // section symbol. If we are here, this means we are probably accessing3166    // data, so it is imperative to keep the original address.3167    if (IsAArch64) {3168      SymbolName = formatv("SYMBOLat{0:x}", Address);3169      SymbolAddress = Address;3170      Addend = 0;3171    }3172 3173    if (BinaryData *BD = BC->getBinaryDataContainingAddress(SymbolAddress)) {3174      // Note: this assertion is trying to check sanity of BinaryData objects3175      // but AArch64 and RISCV has inferred and incomplete object locations3176      // coming from GOT/TLS or any other non-trivial relocation (that requires3177      // creation of sections and whose symbol address is not really what should3178      // be encoded in the instruction). So we essentially disabled this check3179      // for AArch64 and live with bogus names for objects.3180      assert((IsAArch64 || BC->isRISCV() || IsSectionRelocation ||3181              BD->nameStartsWith(SymbolName) ||3182              BD->nameStartsWith("PG" + SymbolName) ||3183              (BD->nameStartsWith("ANONYMOUS") &&3184               (BD->getSectionName().starts_with(".plt") ||3185                BD->getSectionName().ends_with(".plt")))) &&3186             "BOLT symbol names of all non-section relocations must match up "3187             "with symbol names referenced in the relocation");3188 3189      if (IsSectionRelocation)3190        BC->markAmbiguousRelocations(*BD, Address);3191 3192      ReferencedSymbol = BD->getSymbol();3193      Addend += (SymbolAddress - BD->getAddress());3194      SymbolAddress = BD->getAddress();3195      assert(Address == SymbolAddress + Addend);3196    } else {3197      // These are mostly local data symbols but undefined symbols3198      // in relocation sections can get through here too, from .plt.3199      assert(3200          (IsAArch64 || BC->isRISCV() || IsSectionRelocation ||3201           BC->getSectionNameForAddress(SymbolAddress)->starts_with(".plt")) &&3202          "known symbols should not resolve to anonymous locals");3203 3204      if (IsSectionRelocation) {3205        ReferencedSymbol =3206            BC->getOrCreateGlobalSymbol(SymbolAddress, "SYMBOLat");3207      } else {3208        SymbolRef Symbol = *Rel.getSymbol();3209        const uint64_t SymbolSize =3210            IsAArch64 ? 0 : ELFSymbolRef(Symbol).getSize();3211        const uint64_t SymbolAlignment = IsAArch64 ? 1 : Symbol.getAlignment();3212        const uint32_t SymbolFlags = cantFail(Symbol.getFlags());3213        std::string Name;3214        if (SymbolFlags & SymbolRef::SF_Global) {3215          Name = SymbolName;3216        } else {3217          if (StringRef(SymbolName)3218                  .starts_with(BC->AsmInfo->getPrivateGlobalPrefix()))3219            Name = NR.uniquify("PG" + SymbolName);3220          else3221            Name = NR.uniquify(SymbolName);3222        }3223        ReferencedSymbol = BC->registerNameAtAddress(3224            Name, SymbolAddress, SymbolSize, SymbolAlignment, SymbolFlags);3225      }3226 3227      if (IsSectionRelocation) {3228        BinaryData *BD = BC->getBinaryDataByName(ReferencedSymbol->getName());3229        BC->markAmbiguousRelocations(*BD, Address);3230      }3231    }3232  }3233 3234  auto checkMaxDataRelocations = [&]() {3235    ++NumDataRelocations;3236    LLVM_DEBUG(if (opts::MaxDataRelocations &&3237                   NumDataRelocations + 1 == opts::MaxDataRelocations) {3238      dbgs() << "BOLT-DEBUG: processing ending on data relocation "3239             << NumDataRelocations << ": ";3240      printRelocationInfo(Rel, ReferencedSymbol->getName(), SymbolAddress,3241                          Addend, ExtractedValue);3242    });3243 3244    return (!opts::MaxDataRelocations ||3245            NumDataRelocations < opts::MaxDataRelocations);3246  };3247 3248  if ((ReferencedSection && refersToReorderedSection(ReferencedSection)) ||3249      (opts::ForceToDataRelocations && checkMaxDataRelocations()) ||3250      // RISC-V has ADD/SUB data-to-data relocations3251      BC->isRISCV())3252    ForceRelocation = true;3253 3254  if (IsFromCode)3255    ContainingBF->addRelocation(Rel.getOffset(), ReferencedSymbol, RType,3256                                Addend, ExtractedValue);3257  else if (IsToCode || ForceRelocation)3258    BC->addRelocation(Rel.getOffset(), ReferencedSymbol, RType, Addend,3259                      ExtractedValue);3260  else3261    LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocation from data to data\n");3262}3263 3264static BinaryFunction *getInitFunctionIfStaticBinary(BinaryContext &BC) {3265  // Workaround for https://github.com/llvm/llvm-project/issues/1000963266  // ("[BOLT] GOT array pointer incorrectly rewritten"). In aarch643267  // static glibc binaries, the .init section's _init function pointer can3268  // alias with a data pointer for the end of an array. GOT rewriting3269  // currently can't detect this and updates the data pointer to the3270  // moved _init, causing a runtime crash. Skipping _init on the other3271  // hand should be harmless.3272  if (!BC.IsStaticExecutable)3273    return nullptr;3274  const BinaryData *BD = BC.getBinaryDataByName("_init");3275  if (!BD || BD->getSectionName() != ".init")3276    return nullptr;3277  LLVM_DEBUG(dbgs() << "BOLT-DEBUG: skip _init in for GOT workaround.\n");3278  return BC.getBinaryFunctionAtAddress(BD->getAddress());3279}3280 3281static void populateFunctionNames(cl::opt<std::string> &FunctionNamesFile,3282                                  cl::list<std::string> &FunctionNames) {3283  if (FunctionNamesFile.empty())3284    return;3285  std::ifstream FuncsFile(FunctionNamesFile, std::ios::in);3286  std::string FuncName;3287  while (std::getline(FuncsFile, FuncName))3288    FunctionNames.push_back(FuncName);3289}3290 3291void RewriteInstance::selectFunctionsToPrint() {3292  populateFunctionNames(opts::PrintOnlyFile, opts::PrintOnly);3293}3294 3295void RewriteInstance::selectFunctionsToProcess() {3296  // Extend the list of functions to process or skip from a file.3297  populateFunctionNames(opts::FunctionNamesFile, opts::ForceFunctionNames);3298  populateFunctionNames(opts::SkipFunctionNamesFile, opts::SkipFunctionNames);3299  populateFunctionNames(opts::FunctionNamesFileNR, opts::ForceFunctionNamesNR);3300 3301  // Make a set of functions to process to speed up lookups.3302  std::unordered_set<std::string> ForceFunctionsNR(3303      opts::ForceFunctionNamesNR.begin(), opts::ForceFunctionNamesNR.end());3304 3305  if ((!opts::ForceFunctionNames.empty() ||3306       !opts::ForceFunctionNamesNR.empty()) &&3307      !opts::SkipFunctionNames.empty()) {3308    BC->errs()3309        << "BOLT-ERROR: cannot select functions to process and skip at the "3310           "same time. Please use only one type of selection.\n";3311    exit(1);3312  }3313 3314  uint64_t LiteThresholdExecCount = 0;3315  if (opts::LiteThresholdPct) {3316    if (opts::LiteThresholdPct > 100)3317      opts::LiteThresholdPct = 100;3318 3319    std::vector<const BinaryFunction *> TopFunctions;3320    for (auto &BFI : BC->getBinaryFunctions()) {3321      const BinaryFunction &Function = BFI.second;3322      if (ProfileReader->mayHaveProfileData(Function))3323        TopFunctions.push_back(&Function);3324    }3325    llvm::sort(3326        TopFunctions, [](const BinaryFunction *A, const BinaryFunction *B) {3327          return A->getKnownExecutionCount() < B->getKnownExecutionCount();3328        });3329 3330    size_t Index = TopFunctions.size() * opts::LiteThresholdPct / 100;3331    if (Index)3332      --Index;3333    LiteThresholdExecCount = TopFunctions[Index]->getKnownExecutionCount();3334    BC->outs() << "BOLT-INFO: limiting processing to functions with at least "3335               << LiteThresholdExecCount << " invocations\n";3336  }3337  LiteThresholdExecCount = std::max(3338      LiteThresholdExecCount, static_cast<uint64_t>(opts::LiteThresholdCount));3339 3340  StringSet<> ReorderFunctionsUserSet;3341  StringSet<> ReorderFunctionsLTOCommonSet;3342  if (opts::ReorderFunctions == ReorderFunctions::RT_USER) {3343    std::vector<std::string> FunctionNames;3344    BC->logBOLTErrorsAndQuitOnFatal(3345        ReorderFunctions::readFunctionOrderFile(FunctionNames));3346    for (const std::string &Function : FunctionNames) {3347      ReorderFunctionsUserSet.insert(Function);3348      if (std::optional<StringRef> LTOCommonName = getLTOCommonName(Function))3349        ReorderFunctionsLTOCommonSet.insert(*LTOCommonName);3350    }3351  }3352 3353  uint64_t NumFunctionsToProcess = 0;3354  auto mustSkip = [&](const BinaryFunction &Function) {3355    if (opts::MaxFunctions.getNumOccurrences() &&3356        NumFunctionsToProcess >= opts::MaxFunctions)3357      return true;3358    for (std::string &Name : opts::SkipFunctionNames)3359      if (Function.hasNameRegex(Name))3360        return true;3361 3362    return false;3363  };3364 3365  auto shouldProcess = [&](const BinaryFunction &Function) {3366    if (mustSkip(Function))3367      return false;3368 3369    // If the list is not empty, only process functions from the list.3370    if (!opts::ForceFunctionNames.empty() || !ForceFunctionsNR.empty()) {3371      // Regex check (-funcs and -funcs-file options).3372      for (std::string &Name : opts::ForceFunctionNames)3373        if (Function.hasNameRegex(Name))3374          return true;3375 3376      // Non-regex check (-funcs-no-regex and -funcs-file-no-regex).3377      for (const StringRef Name : Function.getNames())3378        if (ForceFunctionsNR.count(Name.str()))3379          return true;3380 3381      return false;3382    }3383 3384    if (opts::Lite) {3385      // Forcibly include functions specified in the -function-order file.3386      if (opts::ReorderFunctions == ReorderFunctions::RT_USER) {3387        for (const StringRef Name : Function.getNames())3388          if (ReorderFunctionsUserSet.contains(Name))3389            return true;3390        for (const StringRef Name : Function.getNames())3391          if (std::optional<StringRef> LTOCommonName = getLTOCommonName(Name))3392            if (ReorderFunctionsLTOCommonSet.contains(*LTOCommonName))3393              return true;3394      }3395 3396      if (ProfileReader && !ProfileReader->mayHaveProfileData(Function))3397        return false;3398 3399      if (Function.getKnownExecutionCount() < LiteThresholdExecCount)3400        return false;3401    }3402 3403    return true;3404  };3405 3406  if (BinaryFunction *Init = getInitFunctionIfStaticBinary(*BC))3407    Init->setIgnored();3408 3409  for (auto &BFI : BC->getBinaryFunctions()) {3410    BinaryFunction &Function = BFI.second;3411 3412    // Pseudo functions are explicitly marked by us not to be processed.3413    if (Function.isPseudo()) {3414      Function.IsIgnored = true;3415      Function.HasExternalRefRelocations = true;3416      continue;3417    }3418 3419    // Decide what to do with fragments after parent functions are processed.3420    if (Function.isFragment())3421      continue;3422 3423    if (!shouldProcess(Function)) {3424      if (opts::Verbosity >= 1) {3425        BC->outs() << "BOLT-INFO: skipping processing " << Function3426                   << " per user request\n";3427      }3428      Function.setIgnored();3429    } else {3430      ++NumFunctionsToProcess;3431      if (opts::MaxFunctions.getNumOccurrences() &&3432          NumFunctionsToProcess == opts::MaxFunctions)3433        BC->outs() << "BOLT-INFO: processing ending on " << Function << '\n';3434    }3435  }3436 3437  if (!BC->HasSplitFunctions)3438    return;3439 3440  // Fragment overrides:3441  // - If the fragment must be skipped, then the parent must be skipped as well.3442  // Otherwise, fragment should follow the parent function:3443  // - if the parent is skipped, skip fragment,3444  // - if the parent is processed, process the fragment(s) as well.3445  for (auto &BFI : BC->getBinaryFunctions()) {3446    BinaryFunction &Function = BFI.second;3447    if (!Function.isFragment())3448      continue;3449    if (mustSkip(Function)) {3450      for (BinaryFunction *Parent : Function.ParentFragments) {3451        if (opts::Verbosity >= 1) {3452          BC->outs() << "BOLT-INFO: skipping processing " << *Parent3453                     << " together with fragment function\n";3454        }3455        Parent->setIgnored();3456        --NumFunctionsToProcess;3457      }3458      Function.setIgnored();3459      continue;3460    }3461 3462    bool IgnoredParent =3463        llvm::any_of(Function.ParentFragments, [&](BinaryFunction *Parent) {3464          return Parent->isIgnored();3465        });3466    if (IgnoredParent) {3467      if (opts::Verbosity >= 1) {3468        BC->outs() << "BOLT-INFO: skipping processing " << Function3469                   << " together with parent function\n";3470      }3471      Function.setIgnored();3472    } else {3473      ++NumFunctionsToProcess;3474      if (opts::Verbosity >= 1) {3475        BC->outs() << "BOLT-INFO: processing " << Function3476                   << " as a sibling of non-ignored function\n";3477      }3478      if (opts::MaxFunctions && NumFunctionsToProcess == opts::MaxFunctions)3479        BC->outs() << "BOLT-INFO: processing ending on " << Function << '\n';3480    }3481  }3482}3483 3484void RewriteInstance::readDebugInfo() {3485  NamedRegionTimer T("readDebugInfo", "read debug info", TimerGroupName,3486                     TimerGroupDesc, opts::TimeRewrite);3487  if (!opts::UpdateDebugSections)3488    return;3489 3490  BC->preprocessDebugInfo();3491}3492 3493void RewriteInstance::preprocessProfileData() {3494  if (!ProfileReader)3495    return;3496 3497  NamedRegionTimer T("preprocessprofile", "pre-process profile data",3498                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);3499 3500  BC->outs() << "BOLT-INFO: pre-processing profile using "3501             << ProfileReader->getReaderName() << '\n';3502 3503  if (BAT->enabledFor(InputFile)) {3504    BC->outs() << "BOLT-INFO: profile collection done on a binary already "3505                  "processed by BOLT\n";3506    ProfileReader->setBAT(&*BAT);3507  }3508 3509  if (Error E = ProfileReader->preprocessProfile(*BC))3510    report_error("cannot pre-process profile", std::move(E));3511 3512  if (!BC->hasSymbolsWithFileName() && ProfileReader->hasLocalsWithFileName() &&3513      !opts::AllowStripped) {3514    BC->errs()3515        << "BOLT-ERROR: input binary does not have local file symbols "3516           "but profile data includes function names with embedded file "3517           "names. It appears that the input binary was stripped while a "3518           "profiled binary was not. If you know what you are doing and "3519           "wish to proceed, use -allow-stripped option.\n";3520    exit(1);3521  }3522}3523 3524void RewriteInstance::initializeMetadataManager() {3525  if (BC->IsLinuxKernel)3526    MetadataManager.registerRewriter(createLinuxKernelRewriter(*BC));3527 3528  MetadataManager.registerRewriter(createBuildIDRewriter(*BC));3529 3530  MetadataManager.registerRewriter(createPseudoProbeRewriter(*BC));3531 3532  MetadataManager.registerRewriter(createRSeqRewriter(*BC));3533 3534  MetadataManager.registerRewriter(createSDTRewriter(*BC));3535 3536  MetadataManager.registerRewriter(createGNUPropertyRewriter(*BC));3537}3538 3539void RewriteInstance::processSectionMetadata() {3540  NamedRegionTimer T("processmetadata-section", "process section metadata",3541                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);3542  initializeMetadataManager();3543 3544  MetadataManager.runSectionInitializers();3545}3546 3547void RewriteInstance::processMetadataPreCFG() {3548  NamedRegionTimer T("processmetadata-precfg", "process metadata pre-CFG",3549                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);3550  MetadataManager.runInitializersPreCFG();3551 3552  processProfileDataPreCFG();3553}3554 3555void RewriteInstance::processMetadataPostCFG() {3556  NamedRegionTimer T("processmetadata-postcfg", "process metadata post-CFG",3557                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);3558  MetadataManager.runInitializersPostCFG();3559}3560 3561void RewriteInstance::processProfileDataPreCFG() {3562  if (!ProfileReader)3563    return;3564 3565  NamedRegionTimer T("processprofile-precfg", "process profile data pre-CFG",3566                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);3567 3568  if (Error E = ProfileReader->readProfilePreCFG(*BC))3569    report_error("cannot read profile pre-CFG", std::move(E));3570}3571 3572void RewriteInstance::processProfileData() {3573  if (!ProfileReader)3574    return;3575 3576  NamedRegionTimer T("processprofile", "process profile data", TimerGroupName,3577                     TimerGroupDesc, opts::TimeRewrite);3578 3579  if (Error E = ProfileReader->readProfile(*BC))3580    report_error("cannot read profile", std::move(E));3581 3582  if (opts::PrintProfile || opts::PrintAll) {3583    for (auto &BFI : BC->getBinaryFunctions()) {3584      BinaryFunction &Function = BFI.second;3585      if (Function.empty())3586        continue;3587 3588      Function.print(BC->outs(), "after attaching profile");3589    }3590  }3591 3592  if (!opts::SaveProfile.empty() && !BAT->enabledFor(InputFile)) {3593    YAMLProfileWriter PW(opts::SaveProfile);3594    PW.writeProfile(*this);3595  }3596  if (opts::AggregateOnly &&3597      opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML &&3598      !BAT->enabledFor(InputFile)) {3599    YAMLProfileWriter PW(opts::OutputFilename);3600    PW.writeProfile(*this);3601  }3602 3603  // Release memory used by profile reader.3604  ProfileReader.reset();3605 3606  if (opts::AggregateOnly) {3607    PrintProgramStats PPS(&*BAT);3608    BC->logBOLTErrorsAndQuitOnFatal(PPS.runOnFunctions(*BC));3609    TimerGroup::printAll(outs());3610    exit(0);3611  }3612}3613 3614void RewriteInstance::disassembleFunctions() {3615  NamedRegionTimer T("disassembleFunctions", "disassemble functions",3616                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);3617  for (auto &BFI : BC->getBinaryFunctions()) {3618    BinaryFunction &Function = BFI.second;3619 3620    ErrorOr<ArrayRef<uint8_t>> FunctionData = Function.getData();3621    if (!FunctionData) {3622      BC->errs() << "BOLT-ERROR: corresponding section is non-executable or "3623                 << "empty for function " << Function << '\n';3624      exit(1);3625    }3626 3627    // Treat zero-sized functions as non-simple ones.3628    if (Function.getSize() == 0) {3629      Function.setSimple(false);3630      continue;3631    }3632 3633    // Offset of the function in the file.3634    const auto *FileBegin =3635        reinterpret_cast<const uint8_t *>(InputFile->getData().data());3636    Function.setFileOffset(FunctionData->begin() - FileBegin);3637 3638    if (!shouldDisassemble(Function)) {3639      NamedRegionTimer T("scan", "scan functions", "buildfuncs",3640                         "Scan Binary Functions", opts::TimeBuild);3641      Function.scanExternalRefs();3642      Function.setSimple(false);3643      continue;3644    }3645 3646    bool DisasmFailed{false};3647    handleAllErrors(Function.disassemble(), [&](const BOLTError &E) {3648      DisasmFailed = true;3649      if (E.isFatal()) {3650        E.log(BC->errs());3651        exit(1);3652      }3653      if (opts::processAllFunctions()) {3654        BC->errs() << BC->generateBugReportMessage(3655            "function cannot be properly disassembled. "3656            "Unable to continue in relocation mode.",3657            Function);3658        exit(1);3659      }3660      if (opts::Verbosity >= 1)3661        BC->outs() << "BOLT-INFO: could not disassemble function " << Function3662                   << ". Will ignore.\n";3663      // Forcefully ignore the function.3664      Function.scanExternalRefs();3665      Function.setIgnored();3666    });3667 3668    if (DisasmFailed)3669      continue;3670 3671    if (opts::PrintAll || opts::PrintDisasm)3672      Function.print(BC->outs(), "after disassembly");3673  }3674 3675  BC->processInterproceduralReferences();3676  BC->populateJumpTables();3677 3678  for (auto &BFI : BC->getBinaryFunctions()) {3679    BinaryFunction &Function = BFI.second;3680 3681    if (!shouldDisassemble(Function))3682      continue;3683 3684    Function.postProcessEntryPoints();3685    Function.postProcessJumpTables();3686  }3687 3688  BC->clearJumpTableTempData();3689  BC->adjustCodePadding();3690 3691  for (auto &BFI : BC->getBinaryFunctions()) {3692    BinaryFunction &Function = BFI.second;3693 3694    if (!shouldDisassemble(Function))3695      continue;3696 3697    if (!Function.isSimple()) {3698      assert((!BC->HasRelocations || Function.getSize() == 0 ||3699              Function.hasIndirectTargetToSplitFragment()) &&3700             "unexpected non-simple function in relocation mode");3701      continue;3702    }3703 3704    // Fill in CFI information for this function3705    if (!Function.trapsOnEntry() && !CFIRdWrt->fillCFIInfoFor(Function)) {3706      if (BC->HasRelocations) {3707        BC->errs() << BC->generateBugReportMessage("unable to fill CFI.",3708                                                   Function);3709        exit(1);3710      } else {3711        BC->errs() << "BOLT-WARNING: unable to fill CFI for function "3712                   << Function << ". Skipping.\n";3713        Function.setSimple(false);3714        continue;3715      }3716    }3717 3718    // Check if fillCFIInfoFor removed any OpNegateRAState CFIs from the3719    // function.3720    if (Function.containedNegateRAState()) {3721      if (!opts::UpdateBranchProtection) {3722        BC->errs()3723            << "BOLT-ERROR: --update-branch-protection is set to false, but "3724            << Function.getPrintName() << " contains .cfi-negate-ra-state\n";3725        exit(1);3726      }3727    }3728 3729    // Parse LSDA.3730    if (Function.getLSDAAddress() != 0 &&3731        !BC->getFragmentsToSkip().count(&Function)) {3732      ErrorOr<BinarySection &> LSDASection =3733          BC->getSectionForAddress(Function.getLSDAAddress());3734      check_error(LSDASection.getError(), "failed to get LSDA section");3735      ArrayRef<uint8_t> LSDAData = ArrayRef<uint8_t>(3736          LSDASection->getData(), LSDASection->getContents().size());3737      BC->logBOLTErrorsAndQuitOnFatal(3738          Function.parseLSDA(LSDAData, LSDASection->getAddress()));3739    }3740  }3741}3742 3743void RewriteInstance::buildFunctionsCFG() {3744  NamedRegionTimer T("buildCFG", "buildCFG", "buildfuncs",3745                     "Build Binary Functions", opts::TimeBuild);3746 3747  // Create annotation indices to allow lock-free execution3748  BC->MIB->getOrCreateAnnotationIndex("JTIndexReg");3749  BC->MIB->getOrCreateAnnotationIndex("NOP");3750 3751  ParallelUtilities::WorkFuncWithAllocTy WorkFun =3752      [&](BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocId) {3753        bool HadErrors{false};3754        handleAllErrors(BF.buildCFG(AllocId), [&](const BOLTError &E) {3755          if (!E.getMessage().empty())3756            E.log(BC->errs());3757          if (E.isFatal())3758            exit(1);3759          HadErrors = true;3760        });3761 3762        if (HadErrors)3763          return;3764 3765        if (opts::PrintAll) {3766          auto L = BC->scopeLock();3767          BF.print(BC->outs(), "while building cfg");3768        }3769      };3770 3771  ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) {3772    return !shouldDisassemble(BF) || !BF.isSimple();3773  };3774 3775  ParallelUtilities::runOnEachFunctionWithUniqueAllocId(3776      *BC, ParallelUtilities::SchedulingPolicy::SP_INST_LINEAR, WorkFun,3777      SkipPredicate, "disassembleFunctions-buildCFG",3778      /*ForceSequential*/ opts::SequentialDisassembly || opts::PrintAll);3779 3780  BC->postProcessSymbolTable();3781}3782 3783void RewriteInstance::postProcessFunctions() {3784  // We mark fragments as non-simple here, not during disassembly,3785  // So we can build their CFGs.3786  BC->skipMarkedFragments();3787  BC->clearFragmentsToSkip();3788 3789  BC->TotalScore = 0;3790  BC->SumExecutionCount = 0;3791  for (auto &BFI : BC->getBinaryFunctions()) {3792    BinaryFunction &Function = BFI.second;3793 3794    // Set function as non-simple if it has dynamic relocations3795    // in constant island, we don't want this function to be optimized3796    // e.g. function splitting is unsupported.3797    if (Function.hasDynamicRelocationAtIsland())3798      Function.setSimple(false);3799 3800    if (Function.empty())3801      continue;3802 3803    Function.postProcessCFG();3804 3805    if (opts::PrintAll || opts::PrintCFG)3806      Function.print(BC->outs(), "after building cfg");3807 3808    if (opts::shouldDumpDot(Function))3809      Function.dumpGraphForPass("00_build-cfg");3810 3811    if (opts::PrintLoopInfo) {3812      Function.calculateLoopInfo();3813      Function.printLoopInfo(BC->outs());3814    }3815 3816    BC->TotalScore += Function.getFunctionScore();3817    BC->SumExecutionCount += Function.getKnownExecutionCount();3818  }3819 3820  if (opts::PrintGlobals) {3821    BC->outs() << "BOLT-INFO: Global symbols:\n";3822    BC->printGlobalSymbols(BC->outs());3823  }3824}3825 3826void RewriteInstance::runOptimizationPasses() {3827  NamedRegionTimer T("runOptimizationPasses", "run optimization passes",3828                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);3829  BC->logBOLTErrorsAndQuitOnFatal(BinaryFunctionPassManager::runAllPasses(*BC));3830}3831 3832void RewriteInstance::runBinaryAnalyses() {3833  NamedRegionTimer T("runBinaryAnalyses", "run binary analysis passes",3834                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);3835  BinaryFunctionPassManager Manager(*BC);3836  // FIXME: add a pass that warns about which functions do not have CFG,3837  // and therefore, analysis is most likely to be less accurate.3838  using GSK = opts::GadgetScannerKind;3839  using PAuthScanner = PAuthGadgetScanner::Analysis;3840 3841  // If no command line option was given, act as if "all" was specified.3842  bool RunAll = !opts::GadgetScannersToRun.getBits() ||3843                opts::GadgetScannersToRun.isSet(GSK::GS_ALL);3844 3845  if (RunAll || opts::GadgetScannersToRun.isSet(GSK::GS_PAUTH)) {3846    Manager.registerPass(3847        std::make_unique<PAuthScanner>(/*OnlyPacRetChecks=*/false));3848  } else if (RunAll || opts::GadgetScannersToRun.isSet(GSK::GS_PACRET)) {3849    Manager.registerPass(3850        std::make_unique<PAuthScanner>(/*OnlyPacRetChecks=*/true));3851  }3852 3853  BC->logBOLTErrorsAndQuitOnFatal(Manager.runPasses());3854}3855 3856void RewriteInstance::preregisterSections() {3857  // Preregister sections before emission to set their order in the output.3858  const unsigned ROFlags = BinarySection::getFlags(/*IsReadOnly*/ true,3859                                                   /*IsText*/ false,3860                                                   /*IsAllocatable*/ true);3861  if (BinarySection *EHFrameSection = getSection(getEHFrameSectionName())) {3862    // New .eh_frame.3863    BC->registerOrUpdateSection(getNewSecPrefix() + getEHFrameSectionName(),3864                                ELF::SHT_PROGBITS, ROFlags);3865    // Fully register a relocatable copy of the original .eh_frame.3866    BC->registerSection(".relocated.eh_frame", *EHFrameSection);3867  }3868  BC->registerOrUpdateSection(getNewSecPrefix() + ".gcc_except_table",3869                              ELF::SHT_PROGBITS, ROFlags);3870  BC->registerOrUpdateSection(getNewSecPrefix() + ".rodata", ELF::SHT_PROGBITS,3871                              ROFlags);3872  BC->registerOrUpdateSection(getNewSecPrefix() + ".rodata.cold",3873                              ELF::SHT_PROGBITS, ROFlags);3874}3875 3876void RewriteInstance::emitAndLink() {3877  NamedRegionTimer T("emitAndLink", "emit and link", TimerGroupName,3878                     TimerGroupDesc, opts::TimeRewrite);3879 3880  SmallString<0> ObjectBuffer;3881  raw_svector_ostream OS(ObjectBuffer);3882 3883  // Implicitly MCObjectStreamer takes ownership of MCAsmBackend (MAB)3884  // and MCCodeEmitter (MCE). ~MCObjectStreamer() will delete these3885  // two instances.3886  std::unique_ptr<MCStreamer> Streamer = BC->createStreamer(OS);3887 3888  if (EHFrameSection) {3889    if (opts::UseOldText || opts::StrictMode) {3890      // The section is going to be regenerated from scratch.3891      // Empty the contents, but keep the section reference.3892      EHFrameSection->clearContents();3893    } else {3894      // Make .eh_frame relocatable.3895      relocateEHFrameSection();3896    }3897  }3898 3899  emitBinaryContext(*Streamer, *BC, getOrgSecPrefix());3900 3901  Streamer->finish();3902  if (Streamer->getContext().hadError()) {3903    BC->errs() << "BOLT-ERROR: Emission failed.\n";3904    exit(1);3905  }3906 3907  if (opts::KeepTmp) {3908    SmallString<128> OutObjectPath;3909    sys::fs::getPotentiallyUniqueTempFileName("output", "o", OutObjectPath);3910    std::error_code EC;3911    raw_fd_ostream FOS(OutObjectPath, EC);3912    check_error(EC, "cannot create output object file");3913    FOS << ObjectBuffer;3914    BC->outs()3915        << "BOLT-INFO: intermediary output object file saved for debugging "3916           "purposes: "3917        << OutObjectPath << "\n";3918  }3919 3920  ErrorOr<BinarySection &> TextSection =3921      BC->getUniqueSectionByName(BC->getMainCodeSectionName());3922  if (BC->HasRelocations && TextSection)3923    BC->renameSection(*TextSection,3924                      getOrgSecPrefix() + BC->getMainCodeSectionName());3925 3926  //////////////////////////////////////////////////////////////////////////////3927  // Assign addresses to new sections.3928  //////////////////////////////////////////////////////////////////////////////3929 3930  // Get output object as ObjectFile.3931  std::unique_ptr<MemoryBuffer> ObjectMemBuffer =3932      MemoryBuffer::getMemBuffer(ObjectBuffer, "in-memory object file", false);3933 3934  auto EFMM = std::make_unique<ExecutableFileMemoryManager>(*BC);3935  EFMM->setNewSecPrefix(getNewSecPrefix());3936  EFMM->setOrgSecPrefix(getOrgSecPrefix());3937 3938  Linker = std::make_unique<JITLinkLinker>(*BC, std::move(EFMM));3939  Linker->loadObject(ObjectMemBuffer->getMemBufferRef(),3940                     [this](auto MapSection) { mapFileSections(MapSection); });3941 3942  // Update output addresses based on the new section map and3943  // layout. Only do this for the object created by ourselves.3944  updateOutputValues(*Linker);3945 3946  if (opts::UpdateDebugSections) {3947    DebugInfoRewriter->updateLineTableOffsets(3948        static_cast<MCObjectStreamer &>(*Streamer).getAssembler());3949  }3950 3951  if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary()) {3952    StartLinkingRuntimeLib = true;3953    RtLibrary->link(*BC, ToolPath, *Linker, [this](auto MapSection) {3954      // Map newly registered sections.3955      this->mapAllocatableSections(MapSection);3956    });3957  }3958 3959  // Once the code is emitted, we can rename function sections to actual3960  // output sections and de-register sections used for emission.3961  for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {3962    ErrorOr<BinarySection &> Section = Function->getCodeSection();3963    if (Section &&3964        (Function->getImageAddress() == 0 || Function->getImageSize() == 0))3965      continue;3966 3967    // Restore origin section for functions that were emitted or supposed to3968    // be emitted to patch sections.3969    if (Section)3970      BC->deregisterSection(*Section);3971    assert(Function->getOriginSectionName() && "expected origin section");3972    Function->CodeSectionName = Function->getOriginSectionName()->str();3973    for (const FunctionFragment &FF :3974         Function->getLayout().getSplitFragments()) {3975      if (ErrorOr<BinarySection &> ColdSection =3976              Function->getCodeSection(FF.getFragmentNum()))3977        BC->deregisterSection(*ColdSection);3978    }3979    if (Function->getLayout().isSplit())3980      Function->setColdCodeSectionName(getBOLTTextSectionName());3981  }3982 3983  if (opts::PrintCacheMetrics) {3984    BC->outs() << "BOLT-INFO: cache metrics after emitting functions:\n";3985    CacheMetrics::printAll(BC->outs(), BC->getSortedFunctions());3986  }3987}3988 3989void RewriteInstance::finalizeMetadataPreEmit() {3990  NamedRegionTimer T("finalizemetadata-preemit", "finalize metadata pre-emit",3991                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);3992  MetadataManager.runFinalizersPreEmit();3993}3994 3995void RewriteInstance::updateMetadata() {3996  NamedRegionTimer T("updatemetadata-postemit", "update metadata post-emit",3997                     TimerGroupName, TimerGroupDesc, opts::TimeRewrite);3998  MetadataManager.runFinalizersAfterEmit();3999 4000  if (opts::UpdateDebugSections) {4001    NamedRegionTimer T("updateDebugInfo", "update debug info", TimerGroupName,4002                       TimerGroupDesc, opts::TimeRewrite);4003    DebugInfoRewriter->updateDebugInfo();4004  }4005 4006  if (opts::WriteBoltInfoSection)4007    addBoltInfoSection();4008}4009 4010void RewriteInstance::mapFileSections(BOLTLinker::SectionMapper MapSection) {4011  BC->deregisterUnusedSections();4012 4013  // If no new .eh_frame was written, remove relocated original .eh_frame.4014  BinarySection *RelocatedEHFrameSection =4015      getSection(".relocated" + getEHFrameSectionName());4016  if (RelocatedEHFrameSection && RelocatedEHFrameSection->hasValidSectionID()) {4017    BinarySection *NewEHFrameSection =4018        getSection(getNewSecPrefix() + getEHFrameSectionName());4019    if (!NewEHFrameSection || !NewEHFrameSection->isFinalized()) {4020      // JITLink will still have to process relocations for the section, hence4021      // we need to assign it the address that wouldn't result in relocation4022      // processing failure.4023      MapSection(*RelocatedEHFrameSection, NextAvailableAddress);4024      BC->deregisterSection(*RelocatedEHFrameSection);4025    }4026  }4027 4028  mapCodeSections(MapSection);4029 4030  // Map the rest of the sections.4031  mapAllocatableSections(MapSection);4032 4033  if (!BC->BOLTReserved.empty()) {4034    const uint64_t AllocatedSize =4035        NextAvailableAddress - BC->BOLTReserved.start();4036    if (BC->BOLTReserved.size() < AllocatedSize) {4037      BC->errs() << "BOLT-ERROR: reserved space (" << BC->BOLTReserved.size()4038                 << " byte" << (BC->BOLTReserved.size() == 1 ? "" : "s")4039                 << ") is smaller than required for new allocations ("4040                 << AllocatedSize << " bytes)\n";4041      exit(1);4042    }4043  }4044}4045 4046std::vector<BinarySection *> RewriteInstance::getCodeSections() {4047  std::vector<BinarySection *> CodeSections;4048  for (BinarySection &Section : BC->textSections())4049    if (Section.hasValidSectionID())4050      CodeSections.emplace_back(&Section);4051 4052  auto compareSections = [&](const BinarySection *A, const BinarySection *B) {4053    // If both A and B have names starting with ".text.cold", then4054    // - if opts::HotFunctionsAtEnd is true, we want order4055    //   ".text.cold.T", ".text.cold.T-1", ... ".text.cold.1", ".text.cold"4056    // - if opts::HotFunctionsAtEnd is false, we want order4057    //   ".text.cold", ".text.cold.1", ... ".text.cold.T-1", ".text.cold.T"4058    if (A->getName().starts_with(BC->getColdCodeSectionName()) &&4059        B->getName().starts_with(BC->getColdCodeSectionName())) {4060      if (A->getName().size() != B->getName().size())4061        return (opts::HotFunctionsAtEnd)4062                   ? (A->getName().size() > B->getName().size())4063                   : (A->getName().size() < B->getName().size());4064      return (opts::HotFunctionsAtEnd) ? (A->getName() > B->getName())4065                                       : (A->getName() < B->getName());4066    }4067 4068    // Place movers before anything else.4069    if (A->getName() == BC->getHotTextMoverSectionName())4070      return true;4071    if (B->getName() == BC->getHotTextMoverSectionName())4072      return false;4073 4074    // Depending on opts::HotFunctionsAtEnd, place main and warm sections in4075    // order.4076    if (opts::HotFunctionsAtEnd) {4077      if (B->getName() == BC->getMainCodeSectionName())4078        return true;4079      if (A->getName() == BC->getMainCodeSectionName())4080        return false;4081      return (B->getName() == BC->getWarmCodeSectionName());4082    } else {4083      if (A->getName() == BC->getMainCodeSectionName())4084        return true;4085      if (B->getName() == BC->getMainCodeSectionName())4086        return false;4087      return (A->getName() == BC->getWarmCodeSectionName());4088    }4089  };4090 4091  // Determine the order of sections.4092  llvm::stable_sort(CodeSections, compareSections);4093 4094  return CodeSections;4095}4096 4097void RewriteInstance::mapCodeSections(BOLTLinker::SectionMapper MapSection) {4098  if (!BC->HasRelocations) {4099    mapCodeSectionsInPlace(MapSection);4100    return;4101  }4102 4103  // Map sections for functions with pre-assigned addresses.4104  for (BinaryFunction *InjectedFunction : BC->getInjectedBinaryFunctions()) {4105    const uint64_t OutputAddress = InjectedFunction->getOutputAddress();4106    if (!OutputAddress)4107      continue;4108 4109    ErrorOr<BinarySection &> FunctionSection =4110        InjectedFunction->getCodeSection();4111    assert(FunctionSection && "function should have section");4112    FunctionSection->setOutputAddress(OutputAddress);4113    MapSection(*FunctionSection, OutputAddress);4114    InjectedFunction->setImageAddress(FunctionSection->getAllocAddress());4115    InjectedFunction->setImageSize(FunctionSection->getOutputSize());4116  }4117 4118  // Populate the list of sections to be allocated.4119  std::vector<BinarySection *> CodeSections = getCodeSections();4120 4121  // Remove sections that were pre-allocated (patch sections).4122  llvm::erase_if(CodeSections, [](BinarySection *Section) {4123    return Section->getOutputAddress();4124  });4125  LLVM_DEBUG(dbgs() << "Code sections in the order of output:\n";4126             for (const BinarySection *Section : CodeSections) dbgs()4127             << Section->getName() << '\n';);4128 4129  uint64_t PaddingSize = 0; // size of padding required at the end4130 4131  // Allocate sections starting at a given Address.4132  auto allocateAt = [&](uint64_t Address) {4133    const char *LastNonColdSectionName = BC->HasWarmSection4134                                             ? BC->getWarmCodeSectionName()4135                                             : BC->getMainCodeSectionName();4136    for (BinarySection *Section : CodeSections) {4137      Address = alignTo(Address, Section->getAlignment());4138      Section->setOutputAddress(Address);4139      Address += Section->getOutputSize();4140 4141      // Hugify: Additional huge page from right side due to4142      // weird ASLR mapping addresses (4KB aligned)4143      if (opts::Hugify && !BC->HasFixedLoadAddress &&4144          Section->getName() == LastNonColdSectionName)4145        Address = alignTo(Address, Section->getAlignment());4146    }4147 4148    // Make sure we allocate enough space for huge pages.4149    ErrorOr<BinarySection &> TextSection =4150        BC->getUniqueSectionByName(LastNonColdSectionName);4151    if (opts::HotText && TextSection && TextSection->hasValidSectionID()) {4152      uint64_t HotTextEnd =4153          TextSection->getOutputAddress() + TextSection->getOutputSize();4154      HotTextEnd = alignTo(HotTextEnd, BC->PageAlign);4155      if (HotTextEnd > Address) {4156        PaddingSize = HotTextEnd - Address;4157        Address = HotTextEnd;4158      }4159    }4160    return Address;4161  };4162 4163  // Try to allocate sections before the \p Address and return an address for4164  // the allocation of the first section, or 0 if [0, Address) range is not4165  // big enough to fit all sections.4166  auto allocateBefore = [&](uint64_t Address) -> uint64_t {4167    for (BinarySection *Section : llvm::reverse(CodeSections)) {4168      if (Section->getOutputSize() > Address)4169        return 0;4170      Address -= Section->getOutputSize();4171      Address = alignDown(Address, Section->getAlignment());4172      Section->setOutputAddress(Address);4173    }4174    return Address;4175  };4176 4177  // Check if we can fit code in the original .text4178  bool AllocationDone = false;4179  if (opts::UseOldText) {4180    uint64_t StartAddress;4181    uint64_t EndAddress;4182    if (opts::HotFunctionsAtEnd) {4183      EndAddress = BC->OldTextSectionAddress + BC->OldTextSectionSize;4184      StartAddress = allocateBefore(EndAddress);4185    } else {4186      StartAddress = BC->OldTextSectionAddress;4187      EndAddress = allocateAt(BC->OldTextSectionAddress);4188    }4189 4190    const uint64_t CodeSize = EndAddress - StartAddress;4191    if (CodeSize <= BC->OldTextSectionSize) {4192      BC->outs() << "BOLT-INFO: using original .text for new code with 0x"4193                 << Twine::utohexstr(opts::AlignText) << " alignment";4194      if (StartAddress != BC->OldTextSectionAddress)4195        BC->outs() << " at 0x" << Twine::utohexstr(StartAddress);4196      BC->outs() << '\n';4197      AllocationDone = true;4198    } else {4199      BC->errs() << "BOLT-WARNING: --use-old-text failed. The original .text "4200                    "too small to fit the new code using 0x"4201                 << Twine::utohexstr(opts::AlignText) << " alignment. "4202                 << CodeSize << " bytes needed, have " << BC->OldTextSectionSize4203                 << " bytes available. Rebuilding without --use-old-text may "4204                    "produce a smaller binary\n";4205      opts::UseOldText = false;4206    }4207  }4208 4209  if (!AllocationDone)4210    NextAvailableAddress = allocateAt(NextAvailableAddress);4211 4212  // Do the mapping for ORC layer based on the allocation.4213  for (BinarySection *Section : CodeSections) {4214    LLVM_DEBUG(dbgs() << "BOLT: mapping " << Section->getName() << " at 0x"4215                      << Twine::utohexstr(Section->getAllocAddress())4216                      << " to 0x"4217                      << Twine::utohexstr(Section->getOutputAddress()) << '\n');4218    MapSection(*Section, Section->getOutputAddress());4219    Section->setOutputFileOffset(4220        getFileOffsetForAddress(Section->getOutputAddress()));4221  }4222 4223  // Check if we need to insert a padding section for hot text.4224  if (PaddingSize && !opts::UseOldText)4225    BC->outs() << "BOLT-INFO: padding code to 0x"4226               << Twine::utohexstr(NextAvailableAddress)4227               << " to accommodate hot text\n";4228}4229 4230void RewriteInstance::mapCodeSectionsInPlace(4231    BOLTLinker::SectionMapper MapSection) {4232  // Processing in non-relocation mode.4233  uint64_t NewTextSectionStartAddress = NextAvailableAddress;4234 4235  for (auto &BFI : BC->getBinaryFunctions()) {4236    BinaryFunction &Function = BFI.second;4237    if (!Function.isEmitted())4238      continue;4239 4240    ErrorOr<BinarySection &> FuncSection = Function.getCodeSection();4241    assert(FuncSection && "cannot find section for function");4242    FuncSection->setOutputAddress(Function.getAddress());4243    LLVM_DEBUG(dbgs() << "BOLT: mapping 0x"4244                      << Twine::utohexstr(FuncSection->getAllocAddress())4245                      << " to 0x" << Twine::utohexstr(Function.getAddress())4246                      << '\n');4247    MapSection(*FuncSection, Function.getAddress());4248    Function.setImageAddress(FuncSection->getAllocAddress());4249    Function.setImageSize(FuncSection->getOutputSize());4250    assert(Function.getImageSize() <= Function.getMaxSize() &&4251           "Unexpected large function");4252 4253    if (!Function.isSplit())4254      continue;4255 4256    assert(Function.getLayout().isHotColdSplit() &&4257           "Cannot allocate more than two fragments per function in "4258           "non-relocation mode.");4259 4260    FunctionFragment &FF =4261        Function.getLayout().getFragment(FragmentNum::cold());4262    ErrorOr<BinarySection &> ColdSection =4263        Function.getCodeSection(FF.getFragmentNum());4264    assert(ColdSection && "cannot find section for cold part");4265    // Cold fragments are aligned at 16 bytes.4266    NextAvailableAddress = alignTo(NextAvailableAddress, 16);4267    FF.setAddress(NextAvailableAddress);4268    FF.setImageAddress(ColdSection->getAllocAddress());4269    FF.setImageSize(ColdSection->getOutputSize());4270    FF.setFileOffset(getFileOffsetForAddress(NextAvailableAddress));4271    ColdSection->setOutputAddress(FF.getAddress());4272 4273    LLVM_DEBUG(4274        dbgs() << formatv(4275            "BOLT: mapping cold fragment {0:x+} to {1:x+} with size {2:x+}\n",4276            FF.getImageAddress(), FF.getAddress(), FF.getImageSize()));4277    MapSection(*ColdSection, FF.getAddress());4278 4279    NextAvailableAddress += FF.getImageSize();4280  }4281 4282  // Add the new text section aggregating all existing code sections.4283  // This is pseudo-section that serves a purpose of creating a corresponding4284  // entry in section header table.4285  const uint64_t NewTextSectionSize =4286      NextAvailableAddress - NewTextSectionStartAddress;4287  if (NewTextSectionSize) {4288    const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true,4289                                                   /*IsText=*/true,4290                                                   /*IsAllocatable=*/true);4291    BinarySection &Section =4292      BC->registerOrUpdateSection(getBOLTTextSectionName(),4293                                  ELF::SHT_PROGBITS,4294                                  Flags,4295                                  /*Data=*/nullptr,4296                                  NewTextSectionSize,4297                                  16);4298    Section.setOutputAddress(NewTextSectionStartAddress);4299    Section.setOutputFileOffset(4300        getFileOffsetForAddress(NewTextSectionStartAddress));4301  }4302}4303 4304void RewriteInstance::mapAllocatableSections(4305    BOLTLinker::SectionMapper MapSection) {4306 4307  if (opts::UseOldText || opts::StrictMode) {4308    auto tryRewriteSection = [&](BinarySection &OldSection,4309                                 BinarySection &NewSection) {4310      if (OldSection.getSize() < NewSection.getOutputSize())4311        return;4312 4313      BC->outs() << "BOLT-INFO: rewriting " << OldSection.getName()4314                 << " in-place\n";4315 4316      NewSection.setOutputAddress(OldSection.getAddress());4317      NewSection.setOutputFileOffset(OldSection.getInputFileOffset());4318      MapSection(NewSection, OldSection.getAddress());4319 4320      // Pad contents with zeros.4321      NewSection.addPadding(OldSection.getSize() - NewSection.getOutputSize());4322 4323      // Prevent the original section name from appearing in the section header4324      // table.4325      OldSection.setAnonymous(true);4326    };4327 4328    if (EHFrameSection) {4329      BinarySection *NewEHFrameSection =4330          getSection(getNewSecPrefix() + getEHFrameSectionName());4331      assert(NewEHFrameSection && "New contents expected for .eh_frame");4332      tryRewriteSection(*EHFrameSection, *NewEHFrameSection);4333    }4334    BinarySection *EHSection = getSection(".gcc_except_table");4335    BinarySection *NewEHSection =4336        getSection(getNewSecPrefix() + ".gcc_except_table");4337    if (EHSection) {4338      assert(NewEHSection && "New contents expected for .gcc_except_table");4339      tryRewriteSection(*EHSection, *NewEHSection);4340    }4341  }4342 4343  // Allocate read-only sections first, then writable sections.4344  enum : uint8_t { ST_READONLY, ST_READWRITE };4345  for (uint8_t SType = ST_READONLY; SType <= ST_READWRITE; ++SType) {4346    const uint64_t LastNextAvailableAddress = NextAvailableAddress;4347    if (SType == ST_READWRITE) {4348      // Align R+W segment to regular page size4349      NextAvailableAddress = alignTo(NextAvailableAddress, BC->RegularPageSize);4350      NewWritableSegmentAddress = NextAvailableAddress;4351    }4352 4353    for (BinarySection &Section : BC->allocatableSections()) {4354      if (Section.isLinkOnly())4355        continue;4356 4357      if (!Section.hasValidSectionID())4358        continue;4359 4360      if (Section.isWritable() == (SType == ST_READONLY))4361        continue;4362 4363      if (Section.getOutputAddress()) {4364        LLVM_DEBUG({4365          dbgs() << "BOLT-DEBUG: section " << Section.getName()4366                 << " is already mapped at 0x"4367                 << Twine::utohexstr(Section.getOutputAddress()) << '\n';4368        });4369        continue;4370      }4371 4372      if (Section.hasSectionRef()) {4373        LLVM_DEBUG({4374          dbgs() << "BOLT-DEBUG: mapping original section " << Section.getName()4375                 << " to 0x" << Twine::utohexstr(Section.getAddress()) << '\n';4376        });4377        Section.setOutputAddress(Section.getAddress());4378        Section.setOutputFileOffset(Section.getInputFileOffset());4379        MapSection(Section, Section.getAddress());4380      } else {4381        uint64_t Alignment = Section.getAlignment();4382        if (opts::Instrument && StartLinkingRuntimeLib) {4383          Alignment = BC->RegularPageSize;4384          StartLinkingRuntimeLib = false;4385        }4386        NextAvailableAddress = alignTo(NextAvailableAddress, Alignment);4387 4388        LLVM_DEBUG({4389          dbgs() << "BOLT-DEBUG: mapping section " << Section.getName()4390                 << " (0x" << Twine::utohexstr(Section.getAllocAddress())4391                 << ") to 0x" << Twine::utohexstr(NextAvailableAddress) << ":0x"4392                 << Twine::utohexstr(NextAvailableAddress +4393                                     Section.getOutputSize())4394                 << '\n';4395        });4396 4397        MapSection(Section, NextAvailableAddress);4398        Section.setOutputAddress(NextAvailableAddress);4399        Section.setOutputFileOffset(4400            getFileOffsetForAddress(NextAvailableAddress));4401 4402        NextAvailableAddress += Section.getOutputSize();4403      }4404    }4405 4406    if (SType == ST_READONLY) {4407      if (NewTextSegmentAddress)4408        NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;4409    } else if (SType == ST_READWRITE) {4410      NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;4411      // Restore NextAvailableAddress if no new writable sections4412      if (!NewWritableSegmentSize)4413        NextAvailableAddress = LastNextAvailableAddress;4414    }4415  }4416}4417 4418void RewriteInstance::updateOutputValues(const BOLTLinker &Linker) {4419  if (std::optional<AddressMap> Map = AddressMap::parse(*BC))4420    BC->setIOAddressMap(std::move(*Map));4421 4422  for (BinaryFunction *Function : BC->getAllBinaryFunctions())4423    Function->updateOutputValues(Linker);4424}4425 4426void RewriteInstance::updateSegmentInfo() {4427  // NOTE Currently .eh_frame_hdr appends to the last segment, recalculate4428  // last segments size based on the NextAvailableAddress variable.4429  if (!NewWritableSegmentSize) {4430    if (NewTextSegmentAddress)4431      NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;4432  } else {4433    NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;4434  }4435 4436  if (NewTextSegmentSize) {4437    SegmentInfo TextSegment = {NewTextSegmentAddress,4438                               NewTextSegmentSize,4439                               NewTextSegmentOffset,4440                               NewTextSegmentSize,4441                               BC->PageAlign,4442                               true,4443                               false};4444    if (!opts::Instrument) {4445      BC->NewSegments.push_back(TextSegment);4446    } else {4447      ErrorOr<BinarySection &> Sec =4448          BC->getUniqueSectionByName(".bolt.instr.counters");4449      assert(Sec && "expected one and only one `.bolt.instr.counters` section");4450      const uint64_t Addr = Sec->getOutputAddress();4451      const uint64_t Offset = Sec->getOutputFileOffset();4452      const uint64_t Size = Sec->getOutputSize();4453      assert(Addr > TextSegment.Address &&4454             Addr + Size < TextSegment.Address + TextSegment.Size &&4455             "`.bolt.instr.counters` section is expected to be included in the "4456             "new text segment");4457 4458      // Set correct size for the previous header since we are breaking the4459      // new text segment into three segments.4460      uint64_t Delta = Addr - TextSegment.Address;4461      TextSegment.Size = Delta;4462      TextSegment.FileSize = Delta;4463      BC->NewSegments.push_back(TextSegment);4464 4465      // Create RW segment that includes the `.bolt.instr.counters` section.4466      SegmentInfo RWSegment = {Addr,  Size, Offset, Size, BC->RegularPageSize,4467                               false, true};4468      BC->NewSegments.push_back(RWSegment);4469 4470      // Create RX segment that includes all RX sections from runtime library.4471      const uint64_t AddrRX = alignTo(Addr + Size, BC->RegularPageSize);4472      const uint64_t OffsetRX = alignTo(Offset + Size, BC->RegularPageSize);4473      const uint64_t SizeRX =4474          NewTextSegmentSize - (AddrRX - TextSegment.Address);4475      SegmentInfo RXSegment = {4476          AddrRX, SizeRX, OffsetRX, SizeRX, BC->RegularPageSize, true, false};4477      BC->NewSegments.push_back(RXSegment);4478    }4479  }4480 4481  if (NewWritableSegmentSize) {4482    SegmentInfo DataSegmentInfo = {4483        NewWritableSegmentAddress,4484        NewWritableSegmentSize,4485        getFileOffsetForAddress(NewWritableSegmentAddress),4486        NewWritableSegmentSize,4487        BC->RegularPageSize,4488        false,4489        true};4490    BC->NewSegments.push_back(DataSegmentInfo);4491  }4492}4493 4494void RewriteInstance::patchELFPHDRTable() {4495  auto ELF64LEFile = cast<ELF64LEObjectFile>(InputFile);4496  const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();4497  raw_fd_ostream &OS = Out->os();4498 4499  Phnum = Obj.getHeader().e_phnum;4500 4501  if (BC->NewSegments.empty()) {4502    BC->outs() << "BOLT-INFO: not adding new segments\n";4503    return;4504  }4505 4506  if (opts::UseGnuStack) {4507    assert(!PHDRTableAddress && "unexpected address for program header table");4508    if (BC->NewSegments.size() > 1) {4509      BC->errs() << "BOLT-ERROR: unable to add writable segment\n";4510      exit(1);4511    }4512  } else {4513    Phnum += BC->NewSegments.size();4514  }4515 4516  if (!PHDRTableOffset)4517    PHDRTableOffset = Obj.getHeader().e_phoff;4518 4519  const uint64_t SavedPos = OS.tell();4520  OS.seek(PHDRTableOffset);4521 4522  auto createPhdr = [](const SegmentInfo &SI) {4523    ELF64LEPhdrTy Phdr;4524    Phdr.p_type = ELF::PT_LOAD;4525    Phdr.p_offset = SI.FileOffset;4526    Phdr.p_vaddr = SI.Address;4527    Phdr.p_paddr = SI.Address;4528    Phdr.p_filesz = SI.FileSize;4529    Phdr.p_memsz = SI.Size;4530    Phdr.p_flags = ELF::PF_R;4531    if (SI.IsExecutable)4532      Phdr.p_flags |= ELF::PF_X;4533    if (SI.IsWritable)4534      Phdr.p_flags |= ELF::PF_W;4535    Phdr.p_align = SI.Alignment;4536 4537    return Phdr;4538  };4539 4540  auto writeNewSegmentPhdrs = [&]() {4541    for (const SegmentInfo &SI : BC->NewSegments) {4542      ELF64LEPhdrTy Phdr = createPhdr(SI);4543      OS.write(reinterpret_cast<const char *>(&Phdr), sizeof(Phdr));4544    }4545  };4546 4547  bool ModdedGnuStack = false;4548  bool AddedSegment = false;4549 4550  // Copy existing program headers with modifications.4551  for (const ELF64LE::Phdr &Phdr : cantFail(Obj.program_headers())) {4552    ELF64LE::Phdr NewPhdr = Phdr;4553    switch (Phdr.p_type) {4554    case ELF::PT_PHDR:4555      if (PHDRTableAddress) {4556        NewPhdr.p_offset = PHDRTableOffset;4557        NewPhdr.p_vaddr = PHDRTableAddress;4558        NewPhdr.p_paddr = PHDRTableAddress;4559        NewPhdr.p_filesz = sizeof(NewPhdr) * Phnum;4560        NewPhdr.p_memsz = sizeof(NewPhdr) * Phnum;4561      }4562      break;4563    case ELF::PT_GNU_EH_FRAME: {4564      ErrorOr<BinarySection &> EHFrameHdrSec = BC->getUniqueSectionByName(4565          getNewSecPrefix() + getEHFrameHdrSectionName());4566      if (EHFrameHdrSec && EHFrameHdrSec->isAllocatable() &&4567          EHFrameHdrSec->isFinalized()) {4568        NewPhdr.p_offset = EHFrameHdrSec->getOutputFileOffset();4569        NewPhdr.p_vaddr = EHFrameHdrSec->getOutputAddress();4570        NewPhdr.p_paddr = EHFrameHdrSec->getOutputAddress();4571        NewPhdr.p_filesz = EHFrameHdrSec->getOutputSize();4572        NewPhdr.p_memsz = EHFrameHdrSec->getOutputSize();4573      }4574      break;4575    }4576    case ELF::PT_GNU_STACK:4577      if (opts::UseGnuStack) {4578        // Overwrite the header with the new segment header.4579        assert(BC->NewSegments.size() == 1 &&4580               "Expected exactly one new segment");4581        NewPhdr = createPhdr(BC->NewSegments.front());4582        ModdedGnuStack = true;4583      }4584      break;4585    case ELF::PT_DYNAMIC:4586      if (!opts::UseGnuStack) {4587        // Insert new headers before DYNAMIC.4588        writeNewSegmentPhdrs();4589        AddedSegment = true;4590      }4591      break;4592    }4593    OS.write(reinterpret_cast<const char *>(&NewPhdr), sizeof(NewPhdr));4594  }4595 4596  if (!opts::UseGnuStack && !AddedSegment) {4597    // Append new headers to the end of the table.4598    writeNewSegmentPhdrs();4599  }4600 4601  if (opts::UseGnuStack && !ModdedGnuStack) {4602    BC->errs()4603        << "BOLT-ERROR: could not find PT_GNU_STACK program header to modify\n";4604    exit(1);4605  }4606 4607  OS.seek(SavedPos);4608}4609 4610namespace {4611 4612/// Write padding to \p OS such that its current \p Offset becomes aligned4613/// at \p Alignment. Return new (aligned) offset.4614uint64_t appendPadding(raw_pwrite_stream &OS, uint64_t Offset,4615                       uint64_t Alignment) {4616  if (!Alignment)4617    return Offset;4618 4619  const uint64_t PaddingSize =4620      offsetToAlignment(Offset, llvm::Align(Alignment));4621  for (unsigned I = 0; I < PaddingSize; ++I)4622    OS.write((unsigned char)0);4623  return Offset + PaddingSize;4624}4625 4626}4627 4628void RewriteInstance::rewriteNoteSections() {4629  auto ELF64LEFile = cast<ELF64LEObjectFile>(InputFile);4630  const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();4631  raw_fd_ostream &OS = Out->os();4632 4633  uint64_t NextAvailableOffset = std::max(4634      getFileOffsetForAddress(NextAvailableAddress), FirstNonAllocatableOffset);4635  OS.seek(NextAvailableOffset);4636 4637  // Copy over non-allocatable section contents and update file offsets.4638  for (const ELF64LE::Shdr &Section : cantFail(Obj.sections())) {4639    if (Section.sh_type == ELF::SHT_NULL)4640      continue;4641    if (Section.sh_flags & ELF::SHF_ALLOC)4642      continue;4643 4644    SectionRef SecRef = ELF64LEFile->toSectionRef(&Section);4645    BinarySection *BSec = BC->getSectionForSectionRef(SecRef);4646    assert(BSec && !BSec->isAllocatable() &&4647           "Matching non-allocatable BinarySection should exist.");4648 4649    StringRef SectionName =4650        cantFail(Obj.getSectionName(Section), "cannot get section name");4651    if (shouldStrip(Section, SectionName))4652      continue;4653 4654    // Insert padding as needed.4655    NextAvailableOffset =4656        appendPadding(OS, NextAvailableOffset, Section.sh_addralign);4657 4658    // New section size.4659    uint64_t Size = 0;4660    bool DataWritten = false;4661    // Copy over section contents unless it's one of the sections we overwrite.4662    if (!willOverwriteSection(SectionName)) {4663      Size = Section.sh_size;4664      StringRef Dataref = InputFile->getData().substr(Section.sh_offset, Size);4665      std::string Data;4666      if (BSec->getPatcher()) {4667        Data = BSec->getPatcher()->patchBinary(Dataref);4668        Dataref = StringRef(Data);4669      }4670 4671      // Section was expanded, so need to treat it as overwrite.4672      if (Size != Dataref.size()) {4673        BSec = &BC->registerOrUpdateNoteSection(4674            SectionName, copyByteArray(Dataref), Dataref.size());4675        Size = 0;4676      } else {4677        OS << Dataref;4678        DataWritten = true;4679 4680        // Add padding as the section extension might rely on the alignment.4681        Size = appendPadding(OS, Size, Section.sh_addralign);4682      }4683    }4684 4685    // Perform section post-processing.4686    assert(BSec->getAlignment() <= Section.sh_addralign &&4687           "alignment exceeds value in file");4688 4689    if (BSec->getAllocAddress()) {4690      assert(!DataWritten && "Writing section twice.");4691      (void)DataWritten;4692      Size += BSec->write(OS);4693    }4694 4695    BSec->setOutputFileOffset(NextAvailableOffset);4696    BSec->flushPendingRelocations(OS, [this](const MCSymbol *S) {4697      return getNewValueForSymbol(S->getName());4698    });4699 4700    // Section contents are no longer needed, but we need to update the size so4701    // that it will be reflected in the section header table.4702    BSec->updateContents(nullptr, Size);4703 4704    NextAvailableOffset += Size;4705  }4706 4707  // Write new note sections.4708  for (BinarySection &Section : BC->nonAllocatableSections()) {4709    if (Section.getOutputFileOffset() || !Section.getAllocAddress())4710      continue;4711 4712    assert(!Section.hasPendingRelocations() && "cannot have pending relocs");4713 4714    NextAvailableOffset =4715        appendPadding(OS, NextAvailableOffset, Section.getAlignment());4716    Section.setOutputFileOffset(NextAvailableOffset);4717 4718    LLVM_DEBUG(4719        dbgs() << "BOLT-DEBUG: writing out new section " << Section.getName()4720               << " of size " << Section.getOutputSize() << " at offset 0x"4721               << Twine::utohexstr(Section.getOutputFileOffset()) << '\n');4722 4723    NextAvailableOffset += Section.write(OS);4724  }4725}4726 4727template <typename ELFT>4728void RewriteInstance::finalizeSectionStringTable(ELFObjectFile<ELFT> *File) {4729  // Pre-populate section header string table.4730  for (const BinarySection &Section : BC->sections())4731    if (!Section.isAnonymous())4732      SHStrTab.add(Section.getOutputName());4733  SHStrTab.finalize();4734 4735  const size_t SHStrTabSize = SHStrTab.getSize();4736  uint8_t *DataCopy = new uint8_t[SHStrTabSize];4737  memset(DataCopy, 0, SHStrTabSize);4738  SHStrTab.write(DataCopy);4739  BC->registerOrUpdateNoteSection(".shstrtab",4740                                  DataCopy,4741                                  SHStrTabSize,4742                                  /*Alignment=*/1,4743                                  /*IsReadOnly=*/true,4744                                  ELF::SHT_STRTAB);4745}4746 4747void RewriteInstance::addBoltInfoSection() {4748  std::string DescStr;4749  raw_string_ostream DescOS(DescStr);4750 4751  DescOS << "BOLT revision: " << BoltRevision << ", "4752         << "command line:";4753  for (int I = 0; I < Argc; ++I)4754    DescOS << " " << Argv[I];4755 4756  // Encode as GNU GOLD VERSION so it is easily printable by 'readelf -n'4757  const std::string BoltInfo =4758      BinarySection::encodeELFNote("GNU", DescStr, 4 /*NT_GNU_GOLD_VERSION*/);4759  BC->registerOrUpdateNoteSection(".note.bolt_info", copyByteArray(BoltInfo),4760                                  BoltInfo.size(),4761                                  /*Alignment=*/1,4762                                  /*IsReadOnly=*/true, ELF::SHT_NOTE);4763}4764 4765void RewriteInstance::addBATSection() {4766  BC->registerOrUpdateNoteSection(BoltAddressTranslation::SECTION_NAME, nullptr,4767                                  0,4768                                  /*Alignment=*/1,4769                                  /*IsReadOnly=*/true, ELF::SHT_NOTE);4770}4771 4772void RewriteInstance::encodeBATSection() {4773  std::string DescStr;4774  raw_string_ostream DescOS(DescStr);4775 4776  BAT->write(*BC, DescOS);4777 4778  const std::string BoltInfo =4779      BinarySection::encodeELFNote("BOLT", DescStr, BinarySection::NT_BOLT_BAT);4780  BC->registerOrUpdateNoteSection(BoltAddressTranslation::SECTION_NAME,4781                                  copyByteArray(BoltInfo), BoltInfo.size(),4782                                  /*Alignment=*/1,4783                                  /*IsReadOnly=*/true, ELF::SHT_NOTE);4784  BC->outs() << "BOLT-INFO: BAT section size (bytes): " << BoltInfo.size()4785             << '\n';4786}4787 4788template <typename ELFShdrTy>4789bool RewriteInstance::shouldStrip(const ELFShdrTy &Section,4790                                  StringRef SectionName) {4791  // Strip non-allocatable relocation sections.4792  if (!(Section.sh_flags & ELF::SHF_ALLOC) && Section.sh_type == ELF::SHT_RELA)4793    return true;4794 4795  // Strip debug sections if not updating them.4796  if (isDebugSection(SectionName) && !opts::UpdateDebugSections)4797    return true;4798 4799  // Strip symtab section if needed4800  if (opts::RemoveSymtab && Section.sh_type == ELF::SHT_SYMTAB)4801    return true;4802 4803  return false;4804}4805 4806template <typename ELFT>4807std::vector<typename object::ELFObjectFile<ELFT>::Elf_Shdr>4808RewriteInstance::getOutputSections(ELFObjectFile<ELFT> *File,4809                                   std::vector<uint32_t> &NewSectionIndex) {4810  using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;4811  const ELFFile<ELFT> &Obj = File->getELFFile();4812  typename ELFT::ShdrRange Sections = cantFail(Obj.sections());4813 4814  // Keep track of section header entries attached to the corresponding section.4815  std::vector<std::pair<BinarySection *, ELFShdrTy>> OutputSections;4816  auto addSection = [&](const ELFShdrTy &Section, BinarySection &BinSec) {4817    ELFShdrTy NewSection = Section;4818    NewSection.sh_name = SHStrTab.getOffset(BinSec.getOutputName());4819    OutputSections.emplace_back(&BinSec, std::move(NewSection));4820  };4821 4822  // Copy over entries for original allocatable sections using modified name.4823  for (const ELFShdrTy &Section : Sections) {4824    // Always ignore this section.4825    if (Section.sh_type == ELF::SHT_NULL) {4826      OutputSections.emplace_back(nullptr, Section);4827      continue;4828    }4829 4830    if (!(Section.sh_flags & ELF::SHF_ALLOC))4831      continue;4832 4833    SectionRef SecRef = File->toSectionRef(&Section);4834    BinarySection *BinSec = BC->getSectionForSectionRef(SecRef);4835    assert(BinSec && "Matching BinarySection should exist.");4836 4837    // Exclude anonymous sections.4838    if (BinSec->isAnonymous())4839      continue;4840 4841    addSection(Section, *BinSec);4842  }4843 4844  for (BinarySection &Section : BC->allocatableSections()) {4845    if (!Section.isFinalized())4846      continue;4847 4848    if (Section.hasSectionRef() || Section.isAnonymous()) {4849      if (opts::Verbosity)4850        BC->outs() << "BOLT-INFO: not writing section header for section "4851                   << Section.getOutputName() << '\n';4852      continue;4853    }4854 4855    if (opts::Verbosity >= 1)4856      BC->outs() << "BOLT-INFO: writing section header for "4857                 << Section.getOutputName() << '\n';4858    ELFShdrTy NewSection;4859    NewSection.sh_type = ELF::SHT_PROGBITS;4860    NewSection.sh_addr = Section.getOutputAddress();4861    NewSection.sh_offset = Section.getOutputFileOffset();4862    NewSection.sh_size = Section.getOutputSize();4863    NewSection.sh_entsize = 0;4864    NewSection.sh_flags = Section.getELFFlags();4865    NewSection.sh_link = 0;4866    NewSection.sh_info = 0;4867    NewSection.sh_addralign = Section.getAlignment();4868    addSection(NewSection, Section);4869  }4870 4871  // Sort all allocatable sections by their offset.4872  llvm::stable_sort(OutputSections, [](const auto &A, const auto &B) {4873    return A.second.sh_offset < B.second.sh_offset;4874  });4875 4876  // Fix section sizes to prevent overlapping.4877  ELFShdrTy *PrevSection = nullptr;4878  BinarySection *PrevBinSec = nullptr;4879  for (auto &SectionKV : OutputSections) {4880    ELFShdrTy &Section = SectionKV.second;4881 4882    // Ignore NOBITS sections as they don't take any space in the file.4883    if (Section.sh_type == ELF::SHT_NOBITS)4884      continue;4885 4886    // Note that address continuity is not guaranteed as sections could be4887    // placed in different loadable segments.4888    if (PrevSection &&4889        PrevSection->sh_offset + PrevSection->sh_size > Section.sh_offset) {4890      if (opts::Verbosity > 1)4891        BC->outs() << "BOLT-INFO: adjusting size for section "4892                   << PrevBinSec->getOutputName() << '\n';4893      PrevSection->sh_size = Section.sh_offset - PrevSection->sh_offset;4894    }4895 4896    PrevSection = &Section;4897    PrevBinSec = SectionKV.first;4898  }4899 4900  uint64_t LastFileOffset = 0;4901 4902  // Copy over entries for non-allocatable sections performing necessary4903  // adjustments.4904  for (const ELFShdrTy &Section : Sections) {4905    if (Section.sh_type == ELF::SHT_NULL)4906      continue;4907    if (Section.sh_flags & ELF::SHF_ALLOC)4908      continue;4909 4910    StringRef SectionName =4911        cantFail(Obj.getSectionName(Section), "cannot get section name");4912 4913    if (shouldStrip(Section, SectionName))4914      continue;4915 4916    SectionRef SecRef = File->toSectionRef(&Section);4917    BinarySection *BinSec = BC->getSectionForSectionRef(SecRef);4918    assert(BinSec && "Matching BinarySection should exist.");4919 4920    ELFShdrTy NewSection = Section;4921    NewSection.sh_offset = BinSec->getOutputFileOffset();4922    NewSection.sh_size = BinSec->getOutputSize();4923 4924    if (NewSection.sh_type == ELF::SHT_SYMTAB)4925      NewSection.sh_info = NumLocalSymbols;4926 4927    addSection(NewSection, *BinSec);4928 4929    LastFileOffset = BinSec->getOutputFileOffset();4930  }4931 4932  // Create entries for new non-allocatable sections.4933  for (BinarySection &Section : BC->nonAllocatableSections()) {4934    if (Section.getOutputFileOffset() <= LastFileOffset)4935      continue;4936 4937    if (opts::Verbosity >= 1)4938      BC->outs() << "BOLT-INFO: writing section header for "4939                 << Section.getOutputName() << '\n';4940 4941    ELFShdrTy NewSection;4942    NewSection.sh_type = Section.getELFType();4943    NewSection.sh_addr = 0;4944    NewSection.sh_offset = Section.getOutputFileOffset();4945    NewSection.sh_size = Section.getOutputSize();4946    NewSection.sh_entsize = 0;4947    NewSection.sh_flags = Section.getELFFlags();4948    NewSection.sh_link = 0;4949    NewSection.sh_info = 0;4950    NewSection.sh_addralign = Section.getAlignment();4951 4952    addSection(NewSection, Section);4953  }4954 4955  // Assign indices to sections.4956  for (uint32_t Index = 1; Index < OutputSections.size(); ++Index)4957    OutputSections[Index].first->setIndex(Index);4958 4959  // Update section index mapping4960  NewSectionIndex.clear();4961  NewSectionIndex.resize(Sections.size(), 0);4962  for (const ELFShdrTy &Section : Sections) {4963    if (Section.sh_type == ELF::SHT_NULL)4964      continue;4965 4966    size_t OrgIndex = std::distance(Sections.begin(), &Section);4967 4968    SectionRef SecRef = File->toSectionRef(&Section);4969    BinarySection *BinSec = BC->getSectionForSectionRef(SecRef);4970    assert(BinSec && "BinarySection should exist for an input section.");4971 4972    // Some sections are stripped4973    if (!BinSec->hasValidIndex())4974      continue;4975 4976    NewSectionIndex[OrgIndex] = BinSec->getIndex();4977  }4978 4979  std::vector<ELFShdrTy> SectionsOnly(OutputSections.size());4980  llvm::copy(llvm::make_second_range(OutputSections), SectionsOnly.begin());4981 4982  return SectionsOnly;4983}4984 4985// Rewrite section header table inserting new entries as needed. The sections4986// header table size itself may affect the offsets of other sections,4987// so we are placing it at the end of the binary.4988//4989// As we rewrite entries we need to track how many sections were inserted4990// as it changes the sh_link value. We map old indices to new ones for4991// existing sections.4992template <typename ELFT>4993void RewriteInstance::patchELFSectionHeaderTable(ELFObjectFile<ELFT> *File) {4994  using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;4995  using ELFEhdrTy = typename ELFObjectFile<ELFT>::Elf_Ehdr;4996  raw_fd_ostream &OS = Out->os();4997  const ELFFile<ELFT> &Obj = File->getELFFile();4998 4999  // Mapping from old section indices to new ones5000  std::vector<uint32_t> NewSectionIndex;5001  std::vector<ELFShdrTy> OutputSections =5002      getOutputSections(File, NewSectionIndex);5003  LLVM_DEBUG(5004    dbgs() << "BOLT-DEBUG: old to new section index mapping:\n";5005    for (uint64_t I = 0; I < NewSectionIndex.size(); ++I)5006      dbgs() << "  " << I << " -> " << NewSectionIndex[I] << '\n';5007  );5008 5009  // Align starting address for section header table. There's no architecutal5010  // need to align this, it is just for pleasant human readability.5011  uint64_t SHTOffset = OS.tell();5012  SHTOffset = appendPadding(OS, SHTOffset, 16);5013 5014  // Write all section header entries while patching section references.5015  for (ELFShdrTy &Section : OutputSections) {5016    Section.sh_link = NewSectionIndex[Section.sh_link];5017    if (Section.sh_type == ELF::SHT_REL || Section.sh_type == ELF::SHT_RELA)5018      Section.sh_info = NewSectionIndex[Section.sh_info];5019    OS.write(reinterpret_cast<const char *>(&Section), sizeof(Section));5020  }5021 5022  // Fix ELF header.5023  ELFEhdrTy NewEhdr = Obj.getHeader();5024 5025  if (BC->HasRelocations) {5026    RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary();5027    if (RtLibrary && opts::RuntimeLibInitHook == opts::RLIH_ENTRY_POINT) {5028      NewEhdr.e_entry = RtLibrary->getRuntimeStartAddress();5029      BC->outs()5030          << "BOLT-INFO: runtime library initialization was hooked via ELF "5031             "Header Entry Point, set to 0x"5032          << Twine::utohexstr(NewEhdr.e_entry) << "\n";5033    } else5034      NewEhdr.e_entry = getNewFunctionAddress(NewEhdr.e_entry);5035    assert((NewEhdr.e_entry || !Obj.getHeader().e_entry) &&5036           "cannot find new address for entry point");5037  }5038  if (PHDRTableOffset) {5039    NewEhdr.e_phoff = PHDRTableOffset;5040    NewEhdr.e_phnum = Phnum;5041  }5042  NewEhdr.e_shoff = SHTOffset;5043  NewEhdr.e_shnum = OutputSections.size();5044  NewEhdr.e_shstrndx = NewSectionIndex[NewEhdr.e_shstrndx];5045  OS.pwrite(reinterpret_cast<const char *>(&NewEhdr), sizeof(NewEhdr), 0);5046}5047 5048template <typename ELFT, typename WriteFuncTy, typename StrTabFuncTy>5049void RewriteInstance::updateELFSymbolTable(5050    ELFObjectFile<ELFT> *File, bool IsDynSym,5051    const typename object::ELFObjectFile<ELFT>::Elf_Shdr &SymTabSection,5052    const std::vector<uint32_t> &NewSectionIndex, WriteFuncTy Write,5053    StrTabFuncTy AddToStrTab) {5054  const ELFFile<ELFT> &Obj = File->getELFFile();5055  using ELFSymTy = typename ELFObjectFile<ELFT>::Elf_Sym;5056 5057  StringRef StringSection =5058      cantFail(Obj.getStringTableForSymtab(SymTabSection));5059 5060  unsigned NumHotTextSymsUpdated = 0;5061  unsigned NumHotDataSymsUpdated = 0;5062 5063  std::map<const BinaryFunction *, uint64_t> IslandSizes;5064  auto getConstantIslandSize = [&IslandSizes](const BinaryFunction &BF) {5065    auto Itr = IslandSizes.find(&BF);5066    if (Itr != IslandSizes.end())5067      return Itr->second;5068    return IslandSizes[&BF] = BF.estimateConstantIslandSize();5069  };5070 5071  // Symbols for the new symbol table.5072  std::vector<ELFSymTy> Symbols;5073 5074  bool EmittedColdFileSymbol = false;5075 5076  auto getNewSectionIndex = [&](uint32_t OldIndex) {5077    // For dynamic symbol table, the section index could be wrong on the input,5078    // and its value is ignored by the runtime if it's different from5079    // SHN_UNDEF and SHN_ABS.5080    // However, we still need to update dynamic symbol table, so return a5081    // section index, even though the index is broken.5082    if (IsDynSym && OldIndex >= NewSectionIndex.size())5083      return OldIndex;5084 5085    assert(OldIndex < NewSectionIndex.size() && "section index out of bounds");5086    const uint32_t NewIndex = NewSectionIndex[OldIndex];5087 5088    // We may have stripped the section that dynsym was referencing due to5089    // the linker bug. In that case return the old index avoiding marking5090    // the symbol as undefined.5091    if (IsDynSym && NewIndex != OldIndex && NewIndex == ELF::SHN_UNDEF)5092      return OldIndex;5093    return NewIndex;5094  };5095 5096  // Get the extra symbol name of a split fragment; used in addExtraSymbols.5097  auto getSplitSymbolName = [&](const FunctionFragment &FF,5098                                const ELFSymTy &FunctionSymbol) {5099    SmallString<256> SymbolName;5100    if (BC->HasWarmSection)5101      SymbolName =5102          formatv("{0}.{1}", cantFail(FunctionSymbol.getName(StringSection)),5103                  FF.getFragmentNum() == FragmentNum::warm() ? "warm" : "cold");5104    else5105      SymbolName = formatv("{0}.cold.{1}",5106                           cantFail(FunctionSymbol.getName(StringSection)),5107                           FF.getFragmentNum().get() - 1);5108    return SymbolName;5109  };5110 5111  // Add extra symbols for the function.5112  //5113  // Note that addExtraSymbols() could be called multiple times for the same5114  // function with different FunctionSymbol matching the main function entry5115  // point.5116  auto addExtraSymbols = [&](const BinaryFunction &Function,5117                             const ELFSymTy &FunctionSymbol) {5118    if (Function.isFolded()) {5119      BinaryFunction *ICFParent = Function.getFoldedIntoFunction();5120      while (ICFParent->isFolded())5121        ICFParent = ICFParent->getFoldedIntoFunction();5122      ELFSymTy ICFSymbol = FunctionSymbol;5123      SmallVector<char, 256> Buf;5124      ICFSymbol.st_name =5125          AddToStrTab(Twine(cantFail(FunctionSymbol.getName(StringSection)))5126                          .concat(".icf.0")5127                          .toStringRef(Buf));5128      ICFSymbol.st_value = ICFParent->getOutputAddress();5129      ICFSymbol.st_size = ICFParent->getOutputSize();5130      ICFSymbol.st_shndx = ICFParent->getCodeSection()->getIndex();5131      Symbols.emplace_back(ICFSymbol);5132    }5133    if (Function.isSplit()) {5134      // Prepend synthetic FILE symbol to prevent local cold fragments from5135      // colliding with existing symbols with the same name.5136      if (!EmittedColdFileSymbol &&5137          FunctionSymbol.getBinding() == ELF::STB_GLOBAL) {5138        ELFSymTy FileSymbol;5139        FileSymbol.st_shndx = ELF::SHN_ABS;5140        FileSymbol.st_name = AddToStrTab(getBOLTFileSymbolName());5141        FileSymbol.st_value = 0;5142        FileSymbol.st_size = 0;5143        FileSymbol.st_other = 0;5144        FileSymbol.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FILE);5145        Symbols.emplace_back(FileSymbol);5146        EmittedColdFileSymbol = true;5147      }5148      for (const FunctionFragment &FF :5149           Function.getLayout().getSplitFragments()) {5150        if (FF.getAddress()) {5151          ELFSymTy NewColdSym = FunctionSymbol;5152          const SmallString<256> SymbolName =5153              getSplitSymbolName(FF, FunctionSymbol);5154          NewColdSym.st_name = AddToStrTab(SymbolName);5155          NewColdSym.st_shndx =5156              Function.getCodeSection(FF.getFragmentNum())->getIndex();5157          NewColdSym.st_value = FF.getAddress();5158          NewColdSym.st_size = FF.getImageSize();5159          NewColdSym.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FUNC);5160          Symbols.emplace_back(NewColdSym);5161        }5162      }5163    }5164    if (Function.hasConstantIsland()) {5165      uint64_t DataMark = Function.getOutputDataAddress();5166      uint64_t CISize = getConstantIslandSize(Function);5167      uint64_t CodeMark = DataMark + CISize;5168      ELFSymTy DataMarkSym = FunctionSymbol;5169      DataMarkSym.st_name = AddToStrTab("$d");5170      DataMarkSym.st_value = DataMark;5171      DataMarkSym.st_size = 0;5172      DataMarkSym.setType(ELF::STT_NOTYPE);5173      DataMarkSym.setBinding(ELF::STB_LOCAL);5174      ELFSymTy CodeMarkSym = DataMarkSym;5175      CodeMarkSym.st_name = AddToStrTab("$x");5176      CodeMarkSym.st_value = CodeMark;5177      Symbols.emplace_back(DataMarkSym);5178      Symbols.emplace_back(CodeMarkSym);5179    }5180    if (Function.hasConstantIsland() && Function.isSplit()) {5181      uint64_t DataMark = Function.getOutputColdDataAddress();5182      uint64_t CISize = getConstantIslandSize(Function);5183      uint64_t CodeMark = DataMark + CISize;5184      ELFSymTy DataMarkSym = FunctionSymbol;5185      DataMarkSym.st_name = AddToStrTab("$d");5186      DataMarkSym.st_value = DataMark;5187      DataMarkSym.st_size = 0;5188      DataMarkSym.setType(ELF::STT_NOTYPE);5189      DataMarkSym.setBinding(ELF::STB_LOCAL);5190      ELFSymTy CodeMarkSym = DataMarkSym;5191      CodeMarkSym.st_name = AddToStrTab("$x");5192      CodeMarkSym.st_value = CodeMark;5193      Symbols.emplace_back(DataMarkSym);5194      Symbols.emplace_back(CodeMarkSym);5195    }5196  };5197 5198  // For regular (non-dynamic) symbol table, exclude symbols referring5199  // to non-allocatable sections.5200  auto shouldStrip = [&](const ELFSymTy &Symbol) {5201    if (Symbol.isAbsolute() || !Symbol.isDefined())5202      return false;5203 5204    // If we cannot link the symbol to a section, leave it as is.5205    Expected<const typename ELFT::Shdr *> Section =5206        Obj.getSection(Symbol.st_shndx);5207    if (!Section)5208      return false;5209 5210    // Remove the section symbol if the corresponding section was stripped.5211    if (Symbol.getType() == ELF::STT_SECTION) {5212      if (!getNewSectionIndex(Symbol.st_shndx))5213        return true;5214      return false;5215    }5216 5217    // Symbols in non-allocatable sections are typically remnants of relocations5218    // emitted under "-emit-relocs" linker option. Delete those as we delete5219    // relocations against non-allocatable sections.5220    if (!((*Section)->sh_flags & ELF::SHF_ALLOC))5221      return true;5222 5223    return false;5224  };5225 5226  for (const ELFSymTy &Symbol : cantFail(Obj.symbols(&SymTabSection))) {5227    // For regular (non-dynamic) symbol table strip unneeded symbols.5228    if (!IsDynSym && shouldStrip(Symbol))5229      continue;5230 5231    const BinaryFunction *Function =5232        BC->getBinaryFunctionAtAddress(Symbol.st_value);5233    // Ignore false function references, e.g. when the section address matches5234    // the address of the function.5235    if (Function && Symbol.getType() == ELF::STT_SECTION)5236      Function = nullptr;5237 5238    // For non-dynamic symtab, make sure the symbol section matches that of5239    // the function. It can mismatch e.g. if the symbol is a section marker5240    // in which case we treat the symbol separately from the function.5241    // For dynamic symbol table, the section index could be wrong on the input,5242    // and its value is ignored by the runtime if it's different from5243    // SHN_UNDEF and SHN_ABS.5244    if (!IsDynSym && Function &&5245        Symbol.st_shndx !=5246            Function->getOriginSection()->getSectionRef().getIndex())5247      Function = nullptr;5248 5249    // Create a new symbol based on the existing symbol.5250    ELFSymTy NewSymbol = Symbol;5251 5252    // Handle special symbols based on their name.5253    Expected<StringRef> SymbolName = Symbol.getName(StringSection);5254    assert(SymbolName && "cannot get symbol name");5255 5256    auto updateSymbolValue = [&](const StringRef Name,5257                                 std::optional<uint64_t> Value = std::nullopt) {5258      NewSymbol.st_value = Value ? *Value : getNewValueForSymbol(Name);5259      NewSymbol.st_shndx = ELF::SHN_ABS;5260      BC->outs() << "BOLT-INFO: setting " << Name << " to 0x"5261                 << Twine::utohexstr(NewSymbol.st_value) << '\n';5262    };5263 5264    if (*SymbolName == "__hot_start" || *SymbolName == "__hot_end") {5265      if (opts::HotText) {5266        updateSymbolValue(*SymbolName);5267        ++NumHotTextSymsUpdated;5268      }5269      goto registerSymbol;5270    }5271 5272    if (*SymbolName == "__hot_data_start" || *SymbolName == "__hot_data_end") {5273      if (opts::HotData) {5274        updateSymbolValue(*SymbolName);5275        ++NumHotDataSymsUpdated;5276      }5277      goto registerSymbol;5278    }5279 5280    if (*SymbolName == "_end") {5281      if (NextAvailableAddress > Symbol.st_value)5282        updateSymbolValue(*SymbolName, NextAvailableAddress);5283      goto registerSymbol;5284    }5285 5286    if (Function) {5287      // If the symbol matched a function that was not emitted, update the5288      // corresponding section index but otherwise leave it unchanged.5289      if (Function->isEmitted()) {5290        NewSymbol.st_value = Function->getOutputAddress();5291        NewSymbol.st_size = Function->getOutputSize();5292        NewSymbol.st_shndx = Function->getCodeSection()->getIndex();5293      } else if (Symbol.st_shndx < ELF::SHN_LORESERVE) {5294        NewSymbol.st_shndx = getNewSectionIndex(Symbol.st_shndx);5295      }5296 5297      // Add new symbols to the symbol table if necessary.5298      if (!IsDynSym)5299        addExtraSymbols(*Function, NewSymbol);5300    } else {5301      // Check if the function symbol matches address inside a function, i.e.5302      // it marks a secondary entry point.5303      Function =5304          (Symbol.getType() == ELF::STT_FUNC)5305              ? BC->getBinaryFunctionContainingAddress(Symbol.st_value,5306                                                       /*CheckPastEnd=*/false,5307                                                       /*UseMaxSize=*/true)5308              : nullptr;5309 5310      if (Function && Function->isEmitted()) {5311        assert(Function->getLayout().isHotColdSplit() &&5312               "Adding symbols based on cold fragment when there are more than "5313               "2 fragments");5314        const uint64_t OutputAddress =5315            Function->translateInputToOutputAddress(Symbol.st_value);5316 5317        NewSymbol.st_value = OutputAddress;5318        // Force secondary entry points to have zero size.5319        NewSymbol.st_size = 0;5320 5321        // Find fragment containing entrypoint5322        FunctionLayout::fragment_const_iterator FF = llvm::find_if(5323            Function->getLayout().fragments(), [&](const FunctionFragment &FF) {5324              uint64_t Lo = FF.getAddress();5325              uint64_t Hi = Lo + FF.getImageSize();5326              return Lo <= OutputAddress && OutputAddress < Hi;5327            });5328 5329        if (FF == Function->getLayout().fragment_end()) {5330          assert(5331              OutputAddress >= Function->getCodeSection()->getOutputAddress() &&5332              OutputAddress < (Function->getCodeSection()->getOutputAddress() +5333                               Function->getCodeSection()->getOutputSize()) &&5334              "Cannot locate fragment containing secondary entrypoint");5335          FF = Function->getLayout().fragment_begin();5336        }5337 5338        NewSymbol.st_shndx =5339            Function->getCodeSection(FF->getFragmentNum())->getIndex();5340      } else {5341        // Check if the symbol belongs to moved data object and update it.5342        BinaryData *BD = opts::ReorderData.empty()5343                             ? nullptr5344                             : BC->getBinaryDataAtAddress(Symbol.st_value);5345        if (BD && BD->isMoved() && !BD->isJumpTable()) {5346          assert((!BD->getSize() || !Symbol.st_size ||5347                  Symbol.st_size == BD->getSize()) &&5348                 "sizes must match");5349 5350          BinarySection &OutputSection = BD->getOutputSection();5351          assert(OutputSection.getIndex());5352          LLVM_DEBUG(dbgs()5353                     << "BOLT-DEBUG: moving " << BD->getName() << " from "5354                     << *BC->getSectionNameForAddress(Symbol.st_value) << " ("5355                     << Symbol.st_shndx << ") to " << OutputSection.getName()5356                     << " (" << OutputSection.getIndex() << ")\n");5357          NewSymbol.st_shndx = OutputSection.getIndex();5358          NewSymbol.st_value = BD->getOutputAddress();5359        } else {5360          // Otherwise just update the section for the symbol.5361          if (Symbol.st_shndx < ELF::SHN_LORESERVE)5362            NewSymbol.st_shndx = getNewSectionIndex(Symbol.st_shndx);5363        }5364 5365        // Detect local syms in the text section that we didn't update5366        // and that were preserved by the linker to support relocations against5367        // .text. Remove them from the symtab.5368        if (Symbol.getType() == ELF::STT_NOTYPE &&5369            Symbol.getBinding() == ELF::STB_LOCAL && Symbol.st_size == 0) {5370          if (BC->getBinaryFunctionContainingAddress(Symbol.st_value,5371                                                     /*CheckPastEnd=*/false,5372                                                     /*UseMaxSize=*/true)) {5373            // Can only delete the symbol if not patching. Such symbols should5374            // not exist in the dynamic symbol table.5375            assert(!IsDynSym && "cannot delete symbol");5376            continue;5377          }5378        }5379      }5380    }5381 5382  registerSymbol:5383    if (IsDynSym)5384      Write((&Symbol - cantFail(Obj.symbols(&SymTabSection)).begin()) *5385                sizeof(ELFSymTy),5386            NewSymbol);5387    else5388      Symbols.emplace_back(NewSymbol);5389  }5390 5391  if (IsDynSym) {5392    assert(Symbols.empty());5393    return;5394  }5395 5396  // Add symbols of injected functions5397  for (BinaryFunction *Function : BC->getInjectedBinaryFunctions()) {5398    if (Function->isAnonymous())5399      continue;5400    ELFSymTy NewSymbol;5401    BinarySection *OriginSection = Function->getOriginSection();5402    NewSymbol.st_shndx =5403        OriginSection5404            ? getNewSectionIndex(OriginSection->getSectionRef().getIndex())5405            : Function->getCodeSection()->getIndex();5406    NewSymbol.st_value = Function->getOutputAddress();5407    NewSymbol.st_name = AddToStrTab(Function->getOneName());5408    NewSymbol.st_size = Function->getOutputSize();5409    NewSymbol.st_other = 0;5410    NewSymbol.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FUNC);5411    Symbols.emplace_back(NewSymbol);5412 5413    if (Function->isSplit()) {5414      assert(Function->getLayout().isHotColdSplit() &&5415             "Adding symbols based on cold fragment when there are more than "5416             "2 fragments");5417      ELFSymTy NewColdSym = NewSymbol;5418      NewColdSym.setType(ELF::STT_NOTYPE);5419      SmallVector<char, 256> Buf;5420      NewColdSym.st_name = AddToStrTab(5421          Twine(Function->getPrintName()).concat(".cold.0").toStringRef(Buf));5422      const FunctionFragment &ColdFF =5423          Function->getLayout().getFragment(FragmentNum::cold());5424      NewColdSym.st_value = ColdFF.getAddress();5425      NewColdSym.st_size = ColdFF.getImageSize();5426      Symbols.emplace_back(NewColdSym);5427    }5428  }5429 5430  auto AddSymbol = [&](const StringRef &Name, uint64_t Address) {5431    if (!Address)5432      return;5433 5434    ELFSymTy Symbol;5435    Symbol.st_value = Address;5436    Symbol.st_shndx = ELF::SHN_ABS;5437    Symbol.st_name = AddToStrTab(Name);5438    Symbol.st_size = 0;5439    Symbol.st_other = 0;5440    Symbol.setBindingAndType(ELF::STB_WEAK, ELF::STT_NOTYPE);5441 5442    BC->outs() << "BOLT-INFO: setting " << Name << " to 0x"5443               << Twine::utohexstr(Symbol.st_value) << '\n';5444 5445    Symbols.emplace_back(Symbol);5446  };5447 5448  // Add runtime library start and fini address symbols5449  if (RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary()) {5450    AddSymbol("__bolt_runtime_start", RtLibrary->getRuntimeStartAddress());5451    AddSymbol("__bolt_runtime_fini", RtLibrary->getRuntimeFiniAddress());5452  }5453 5454  assert((!NumHotTextSymsUpdated || NumHotTextSymsUpdated == 2) &&5455         "either none or both __hot_start/__hot_end symbols were expected");5456  assert((!NumHotDataSymsUpdated || NumHotDataSymsUpdated == 2) &&5457         "either none or both __hot_data_start/__hot_data_end symbols were "5458         "expected");5459 5460  auto AddEmittedSymbol = [&](const StringRef &Name) {5461    AddSymbol(Name, getNewValueForSymbol(Name));5462  };5463 5464  if (opts::HotText && !NumHotTextSymsUpdated) {5465    AddEmittedSymbol("__hot_start");5466    AddEmittedSymbol("__hot_end");5467  }5468 5469  if (opts::HotData && !NumHotDataSymsUpdated) {5470    AddEmittedSymbol("__hot_data_start");5471    AddEmittedSymbol("__hot_data_end");5472  }5473 5474  // Put local symbols at the beginning.5475  llvm::stable_sort(Symbols, [](const ELFSymTy &A, const ELFSymTy &B) {5476    if (A.getBinding() == ELF::STB_LOCAL && B.getBinding() != ELF::STB_LOCAL)5477      return true;5478    return false;5479  });5480 5481  for (const ELFSymTy &Symbol : Symbols)5482    Write(0, Symbol);5483}5484 5485template <typename ELFT>5486void RewriteInstance::patchELFSymTabs(ELFObjectFile<ELFT> *File) {5487  const ELFFile<ELFT> &Obj = File->getELFFile();5488  using ELFShdrTy = typename ELFObjectFile<ELFT>::Elf_Shdr;5489  using ELFSymTy = typename ELFObjectFile<ELFT>::Elf_Sym;5490 5491  // Compute a preview of how section indices will change after rewriting, so5492  // we can properly update the symbol table based on new section indices.5493  std::vector<uint32_t> NewSectionIndex;5494  getOutputSections(File, NewSectionIndex);5495 5496  // Update dynamic symbol table.5497  const ELFShdrTy *DynSymSection = nullptr;5498  for (const ELFShdrTy &Section : cantFail(Obj.sections())) {5499    if (Section.sh_type == ELF::SHT_DYNSYM) {5500      DynSymSection = &Section;5501      break;5502    }5503  }5504  assert((DynSymSection || BC->IsStaticExecutable) &&5505         "dynamic symbol table expected");5506  if (DynSymSection) {5507    updateELFSymbolTable(5508        File,5509        /*IsDynSym=*/true,5510        *DynSymSection,5511        NewSectionIndex,5512        [&](size_t Offset, const ELFSymTy &Sym) {5513          Out->os().pwrite(reinterpret_cast<const char *>(&Sym),5514                           sizeof(ELFSymTy),5515                           DynSymSection->sh_offset + Offset);5516        },5517        [](StringRef) -> size_t { return 0; });5518  }5519 5520  if (opts::RemoveSymtab)5521    return;5522 5523  // (re)create regular symbol table.5524  const ELFShdrTy *SymTabSection = nullptr;5525  for (const ELFShdrTy &Section : cantFail(Obj.sections())) {5526    if (Section.sh_type == ELF::SHT_SYMTAB) {5527      SymTabSection = &Section;5528      break;5529    }5530  }5531  if (!SymTabSection) {5532    BC->errs() << "BOLT-WARNING: no symbol table found\n";5533    return;5534  }5535 5536  const ELFShdrTy *StrTabSection =5537      cantFail(Obj.getSection(SymTabSection->sh_link));5538  std::string NewContents;5539  std::string NewStrTab = std::string(5540      File->getData().substr(StrTabSection->sh_offset, StrTabSection->sh_size));5541  StringRef SecName = cantFail(Obj.getSectionName(*SymTabSection));5542  StringRef StrSecName = cantFail(Obj.getSectionName(*StrTabSection));5543 5544  NumLocalSymbols = 0;5545  updateELFSymbolTable(5546      File,5547      /*IsDynSym=*/false,5548      *SymTabSection,5549      NewSectionIndex,5550      [&](size_t Offset, const ELFSymTy &Sym) {5551        if (Sym.getBinding() == ELF::STB_LOCAL)5552          ++NumLocalSymbols;5553        NewContents.append(reinterpret_cast<const char *>(&Sym),5554                           sizeof(ELFSymTy));5555      },5556      [&](StringRef Str) {5557        size_t Idx = NewStrTab.size();5558        NewStrTab.append(NameResolver::restore(Str).str());5559        NewStrTab.append(1, '\0');5560        return Idx;5561      });5562 5563  BC->registerOrUpdateNoteSection(SecName,5564                                  copyByteArray(NewContents),5565                                  NewContents.size(),5566                                  /*Alignment=*/1,5567                                  /*IsReadOnly=*/true,5568                                  ELF::SHT_SYMTAB);5569 5570  BC->registerOrUpdateNoteSection(StrSecName,5571                                  copyByteArray(NewStrTab),5572                                  NewStrTab.size(),5573                                  /*Alignment=*/1,5574                                  /*IsReadOnly=*/true,5575                                  ELF::SHT_STRTAB);5576}5577 5578template <typename ELFT>5579void RewriteInstance::patchELFAllocatableRelrSection(5580    ELFObjectFile<ELFT> *File) {5581  if (!DynamicRelrAddress)5582    return;5583 5584  raw_fd_ostream &OS = Out->os();5585  const uint8_t PSize = BC->AsmInfo->getCodePointerSize();5586  const uint64_t MaxDelta = ((CHAR_BIT * DynamicRelrEntrySize) - 1) * PSize;5587 5588  auto FixAddend = [&](const BinarySection &Section, const Relocation &Rel,5589                       uint64_t FileOffset) {5590    // Fix relocation symbol value in place if no static relocation found5591    // on the same address. We won't check the BF relocations here since it5592    // is rare case and no optimization is required.5593    if (Section.getRelocationAt(Rel.Offset))5594      return;5595 5596    // No fixup needed if symbol address was not changed5597    const uint64_t Addend = getNewFunctionOrDataAddress(Rel.Addend);5598    if (!Addend)5599      return;5600 5601    OS.pwrite(reinterpret_cast<const char *>(&Addend), PSize, FileOffset);5602  };5603 5604  // Fill new relative relocation offsets set5605  std::set<uint64_t> RelOffsets;5606  for (const BinarySection &Section : BC->allocatableSections()) {5607    const uint64_t SectionInputAddress = Section.getAddress();5608    uint64_t SectionAddress = Section.getOutputAddress();5609    if (!SectionAddress)5610      SectionAddress = SectionInputAddress;5611 5612    for (const Relocation &Rel : Section.dynamicRelocations()) {5613      if (!Rel.isRelative())5614        continue;5615 5616      uint64_t RelOffset =5617          getNewFunctionOrDataAddress(SectionInputAddress + Rel.Offset);5618 5619      RelOffset = RelOffset == 0 ? SectionAddress + Rel.Offset : RelOffset;5620      assert((RelOffset & 1) == 0 && "Wrong relocation offset");5621      RelOffsets.emplace(RelOffset);5622      FixAddend(Section, Rel, RelOffset);5623    }5624  }5625 5626  ErrorOr<BinarySection &> Section =5627      BC->getSectionForAddress(*DynamicRelrAddress);5628  assert(Section && "cannot get .relr.dyn section");5629  assert(Section->isRelr() && "Expected section to be SHT_RELR type");5630  uint64_t RelrDynOffset = Section->getInputFileOffset();5631  const uint64_t RelrDynEndOffset = RelrDynOffset + Section->getSize();5632 5633  auto WriteRelr = [&](uint64_t Value) {5634    if (RelrDynOffset + DynamicRelrEntrySize > RelrDynEndOffset) {5635      BC->errs() << "BOLT-ERROR: Offset overflow for relr.dyn section\n";5636      exit(1);5637    }5638 5639    OS.pwrite(reinterpret_cast<const char *>(&Value), DynamicRelrEntrySize,5640              RelrDynOffset);5641    RelrDynOffset += DynamicRelrEntrySize;5642  };5643 5644  for (auto RelIt = RelOffsets.begin(); RelIt != RelOffsets.end();) {5645    WriteRelr(*RelIt);5646    uint64_t Base = *RelIt++ + PSize;5647    while (1) {5648      uint64_t Bitmap = 0;5649      for (; RelIt != RelOffsets.end(); ++RelIt) {5650        const uint64_t Delta = *RelIt - Base;5651        if (Delta >= MaxDelta || Delta % PSize)5652          break;5653 5654        Bitmap |= (1ULL << (Delta / PSize));5655      }5656 5657      if (!Bitmap)5658        break;5659 5660      WriteRelr((Bitmap << 1) | 1);5661      Base += MaxDelta;5662    }5663  }5664 5665  // Fill the rest of the section with empty bitmap value5666  while (RelrDynOffset != RelrDynEndOffset)5667    WriteRelr(1);5668}5669 5670template <typename ELFT>5671void5672RewriteInstance::patchELFAllocatableRelaSections(ELFObjectFile<ELFT> *File) {5673  using Elf_Rela = typename ELFT::Rela;5674  raw_fd_ostream &OS = Out->os();5675  const ELFFile<ELFT> &EF = File->getELFFile();5676 5677  uint64_t RelDynOffset = 0, RelDynEndOffset = 0;5678  uint64_t RelPltOffset = 0, RelPltEndOffset = 0;5679 5680  auto setSectionFileOffsets = [&](uint64_t Address, uint64_t &Start,5681                                   uint64_t &End) {5682    ErrorOr<BinarySection &> Section = BC->getSectionForAddress(Address);5683    assert(Section && "cannot get relocation section");5684    Start = Section->getInputFileOffset();5685    End = Start + Section->getSize();5686  };5687 5688  if (!DynamicRelocationsAddress && !PLTRelocationsAddress)5689    return;5690 5691  if (DynamicRelocationsAddress)5692    setSectionFileOffsets(*DynamicRelocationsAddress, RelDynOffset,5693                          RelDynEndOffset);5694 5695  if (PLTRelocationsAddress)5696    setSectionFileOffsets(*PLTRelocationsAddress, RelPltOffset,5697                          RelPltEndOffset);5698 5699  DynamicRelativeRelocationsCount = 0;5700 5701  auto writeRela = [&OS](const Elf_Rela *RelA, uint64_t &Offset) {5702    OS.pwrite(reinterpret_cast<const char *>(RelA), sizeof(*RelA), Offset);5703    Offset += sizeof(*RelA);5704  };5705 5706  auto writeRelocations = [&](bool PatchRelative) {5707    for (BinarySection &Section : BC->allocatableSections()) {5708      const uint64_t SectionInputAddress = Section.getAddress();5709      uint64_t SectionAddress = Section.getOutputAddress();5710      if (!SectionAddress)5711        SectionAddress = SectionInputAddress;5712 5713      for (const Relocation &Rel : Section.dynamicRelocations()) {5714        const bool IsRelative = Rel.isRelative();5715        if (PatchRelative != IsRelative)5716          continue;5717 5718        if (IsRelative)5719          ++DynamicRelativeRelocationsCount;5720 5721        Elf_Rela NewRelA;5722        MCSymbol *Symbol = Rel.Symbol;5723        uint32_t SymbolIdx = 0;5724        uint64_t Addend = Rel.Addend;5725        uint64_t RelOffset =5726            getNewFunctionOrDataAddress(SectionInputAddress + Rel.Offset);5727 5728        RelOffset = RelOffset == 0 ? SectionAddress + Rel.Offset : RelOffset;5729        if (Rel.Symbol) {5730          SymbolIdx = getOutputDynamicSymbolIndex(Symbol);5731        } else {5732          // Usually this case is used for R_*_(I)RELATIVE relocations5733          const uint64_t Address = getNewFunctionOrDataAddress(Addend);5734          if (Address)5735            Addend = Address;5736        }5737 5738        NewRelA.setSymbolAndType(SymbolIdx, Rel.Type, EF.isMips64EL());5739        NewRelA.r_offset = RelOffset;5740        NewRelA.r_addend = Addend;5741 5742        const bool IsJmpRel = IsJmpRelocation.contains(Rel.Type);5743        uint64_t &Offset = IsJmpRel ? RelPltOffset : RelDynOffset;5744        const uint64_t &EndOffset =5745            IsJmpRel ? RelPltEndOffset : RelDynEndOffset;5746        if (!Offset || !EndOffset) {5747          BC->errs() << "BOLT-ERROR: Invalid offsets for dynamic relocation\n";5748          exit(1);5749        }5750 5751        if (Offset + sizeof(NewRelA) > EndOffset) {5752          BC->errs() << "BOLT-ERROR: Offset overflow for dynamic relocation\n";5753          exit(1);5754        }5755 5756        writeRela(&NewRelA, Offset);5757      }5758    }5759  };5760 5761  // Place R_*_RELATIVE relocations in RELA section if RELR is not presented.5762  // The dynamic linker expects all R_*_RELATIVE relocations in RELA5763  // to be emitted first.5764  if (!DynamicRelrAddress)5765    writeRelocations(/* PatchRelative */ true);5766  writeRelocations(/* PatchRelative */ false);5767 5768  auto fillNone = [&](uint64_t &Offset, uint64_t EndOffset) {5769    if (!Offset)5770      return;5771 5772    typename ELFObjectFile<ELFT>::Elf_Rela RelA;5773    RelA.setSymbolAndType(0, Relocation::getNone(), EF.isMips64EL());5774    RelA.r_offset = 0;5775    RelA.r_addend = 0;5776    while (Offset < EndOffset)5777      writeRela(&RelA, Offset);5778 5779    assert(Offset == EndOffset && "Unexpected section overflow");5780  };5781 5782  // Fill the rest of the sections with R_*_NONE relocations5783  fillNone(RelDynOffset, RelDynEndOffset);5784  fillNone(RelPltOffset, RelPltEndOffset);5785}5786 5787template <typename ELFT>5788void RewriteInstance::patchELFGOT(ELFObjectFile<ELFT> *File) {5789  raw_fd_ostream &OS = Out->os();5790 5791  SectionRef GOTSection;5792  for (const SectionRef &Section : File->sections()) {5793    StringRef SectionName = cantFail(Section.getName());5794    if (SectionName == ".got") {5795      GOTSection = Section;5796      break;5797    }5798  }5799  if (!GOTSection.getObject()) {5800    if (!BC->IsStaticExecutable)5801      BC->errs() << "BOLT-INFO: no .got section found\n";5802    return;5803  }5804 5805  StringRef GOTContents = cantFail(GOTSection.getContents());5806  for (const uint64_t *GOTEntry =5807           reinterpret_cast<const uint64_t *>(GOTContents.data());5808       GOTEntry < reinterpret_cast<const uint64_t *>(GOTContents.data() +5809                                                     GOTContents.size());5810       ++GOTEntry) {5811    if (uint64_t NewAddress = getNewFunctionAddress(*GOTEntry)) {5812      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching GOT entry 0x"5813                        << Twine::utohexstr(*GOTEntry) << " with 0x"5814                        << Twine::utohexstr(NewAddress) << '\n');5815      OS.pwrite(reinterpret_cast<const char *>(&NewAddress), sizeof(NewAddress),5816                reinterpret_cast<const char *>(GOTEntry) -5817                    File->getData().data());5818    }5819  }5820}5821 5822template <typename ELFT>5823void RewriteInstance::patchELFDynamic(ELFObjectFile<ELFT> *File) {5824  if (BC->IsStaticExecutable)5825    return;5826 5827  const ELFFile<ELFT> &Obj = File->getELFFile();5828  raw_fd_ostream &OS = Out->os();5829 5830  using Elf_Phdr = typename ELFFile<ELFT>::Elf_Phdr;5831  using Elf_Dyn = typename ELFFile<ELFT>::Elf_Dyn;5832 5833  // Locate DYNAMIC by looking through program headers.5834  uint64_t DynamicOffset = 0;5835  const Elf_Phdr *DynamicPhdr = nullptr;5836  for (const Elf_Phdr &Phdr : cantFail(Obj.program_headers())) {5837    if (Phdr.p_type == ELF::PT_DYNAMIC) {5838      DynamicOffset = Phdr.p_offset;5839      DynamicPhdr = &Phdr;5840      assert(Phdr.p_memsz == Phdr.p_filesz && "dynamic sizes should match");5841      break;5842    }5843  }5844  assert(DynamicPhdr && "missing dynamic in ELF binary");5845 5846  bool ZNowSet = false;5847 5848  // Go through all dynamic entries and patch functions addresses with5849  // new ones.5850  typename ELFT::DynRange DynamicEntries =5851      cantFail(Obj.dynamicEntries(), "error accessing dynamic table");5852  auto DTB = DynamicEntries.begin();5853  for (const Elf_Dyn &Dyn : DynamicEntries) {5854    Elf_Dyn NewDE = Dyn;5855    bool ShouldPatch = true;5856    switch (Dyn.d_tag) {5857    default:5858      ShouldPatch = false;5859      break;5860    case ELF::DT_RELACOUNT:5861      NewDE.d_un.d_val = DynamicRelativeRelocationsCount;5862      break;5863    case ELF::DT_INIT:5864    case ELF::DT_FINI: {5865      if (BC->HasRelocations) {5866        if (uint64_t NewAddress = getNewFunctionAddress(Dyn.getPtr())) {5867          LLVM_DEBUG(dbgs() << "BOLT-DEBUG: patching dynamic entry of type "5868                            << Dyn.getTag() << '\n');5869          NewDE.d_un.d_ptr = NewAddress;5870        }5871      }5872      RuntimeLibrary *RtLibrary = BC->getRuntimeLibrary();5873      if (RtLibrary && Dyn.getTag() == ELF::DT_FINI) {5874        if (uint64_t Addr = RtLibrary->getRuntimeFiniAddress()) {5875          NewDE.d_un.d_ptr = Addr;5876          BC->outs()5877              << "BOLT-INFO: runtime library finalization was hooked via "5878                 "DT_FINI, set to 0x"5879              << Twine::utohexstr(Addr) << "\n";5880        }5881      }5882      if (RtLibrary && Dyn.getTag() == ELF::DT_INIT &&5883          (!BC->HasInterpHeader ||5884           opts::RuntimeLibInitHook == opts::RLIH_INIT)) {5885        if (auto Addr = RtLibrary->getRuntimeStartAddress()) {5886          NewDE.d_un.d_ptr = Addr;5887          BC->outs()5888              << "BOLT-INFO: runtime library initialization was hooked via "5889                 "DT_INIT, set to 0x"5890              << Twine::utohexstr(Addr) << "\n";5891        }5892      }5893      break;5894    }5895    case ELF::DT_FLAGS:5896      if (BC->RequiresZNow) {5897        NewDE.d_un.d_val |= ELF::DF_BIND_NOW;5898        ZNowSet = true;5899      }5900      break;5901    case ELF::DT_FLAGS_1:5902      if (BC->RequiresZNow) {5903        NewDE.d_un.d_val |= ELF::DF_1_NOW;5904        ZNowSet = true;5905      }5906      break;5907    }5908    if (ShouldPatch)5909      OS.pwrite(reinterpret_cast<const char *>(&NewDE), sizeof(NewDE),5910                DynamicOffset + (&Dyn - DTB) * sizeof(Dyn));5911  }5912 5913  if (BC->RequiresZNow && !ZNowSet) {5914    BC->errs()5915        << "BOLT-ERROR: output binary requires immediate relocation "5916           "processing which depends on DT_FLAGS or DT_FLAGS_1 presence in "5917           ".dynamic. Please re-link the binary with -znow.\n";5918    exit(1);5919  }5920}5921 5922template <typename ELFT>5923Error RewriteInstance::readELFDynamic(ELFObjectFile<ELFT> *File) {5924  const ELFFile<ELFT> &Obj = File->getELFFile();5925 5926  using Elf_Phdr = typename ELFFile<ELFT>::Elf_Phdr;5927  using Elf_Dyn = typename ELFFile<ELFT>::Elf_Dyn;5928 5929  // Locate DYNAMIC by looking through program headers.5930  const Elf_Phdr *DynamicPhdr = nullptr;5931  for (const Elf_Phdr &Phdr : cantFail(Obj.program_headers())) {5932    if (Phdr.p_type == ELF::PT_DYNAMIC) {5933      DynamicPhdr = &Phdr;5934      break;5935    }5936  }5937 5938  if (!DynamicPhdr) {5939    BC->outs() << "BOLT-INFO: static input executable detected\n";5940    // TODO: static PIE executable might have dynamic header5941    BC->IsStaticExecutable = true;5942    return Error::success();5943  }5944 5945  if (DynamicPhdr->p_memsz != DynamicPhdr->p_filesz)5946    return createStringError(errc::executable_format_error,5947                             "dynamic section sizes should match");5948 5949  // Go through all dynamic entries to locate entries of interest.5950  auto DynamicEntriesOrErr = Obj.dynamicEntries();5951  if (!DynamicEntriesOrErr)5952    return DynamicEntriesOrErr.takeError();5953  typename ELFT::DynRange DynamicEntries = DynamicEntriesOrErr.get();5954 5955  for (const Elf_Dyn &Dyn : DynamicEntries) {5956    switch (Dyn.d_tag) {5957    case ELF::DT_INIT:5958      BC->InitAddress = Dyn.getPtr();5959      break;5960    case ELF::DT_INIT_ARRAY:5961      BC->InitArrayAddress = Dyn.getPtr();5962      break;5963    case ELF::DT_INIT_ARRAYSZ:5964      BC->InitArraySize = Dyn.getPtr();5965      break;5966    case ELF::DT_FINI:5967      BC->FiniAddress = Dyn.getPtr();5968      break;5969    case ELF::DT_FINI_ARRAY:5970      BC->FiniArrayAddress = Dyn.getPtr();5971      break;5972    case ELF::DT_FINI_ARRAYSZ:5973      BC->FiniArraySize = Dyn.getPtr();5974      break;5975    case ELF::DT_RELA:5976      DynamicRelocationsAddress = Dyn.getPtr();5977      break;5978    case ELF::DT_RELASZ:5979      DynamicRelocationsSize = Dyn.getVal();5980      break;5981    case ELF::DT_JMPREL:5982      PLTRelocationsAddress = Dyn.getPtr();5983      break;5984    case ELF::DT_PLTRELSZ:5985      PLTRelocationsSize = Dyn.getVal();5986      break;5987    case ELF::DT_RELACOUNT:5988      DynamicRelativeRelocationsCount = Dyn.getVal();5989      break;5990    case ELF::DT_RELR:5991      DynamicRelrAddress = Dyn.getPtr();5992      break;5993    case ELF::DT_RELRSZ:5994      DynamicRelrSize = Dyn.getVal();5995      break;5996    case ELF::DT_RELRENT:5997      DynamicRelrEntrySize = Dyn.getVal();5998      break;5999    }6000  }6001 6002  if (!DynamicRelocationsAddress || !DynamicRelocationsSize) {6003    DynamicRelocationsAddress.reset();6004    DynamicRelocationsSize = 0;6005  }6006 6007  if (!PLTRelocationsAddress || !PLTRelocationsSize) {6008    PLTRelocationsAddress.reset();6009    PLTRelocationsSize = 0;6010  }6011 6012  if (!DynamicRelrAddress || !DynamicRelrSize) {6013    DynamicRelrAddress.reset();6014    DynamicRelrSize = 0;6015  } else if (!DynamicRelrEntrySize) {6016    BC->errs() << "BOLT-ERROR: expected DT_RELRENT to be presented "6017               << "in DYNAMIC section\n";6018    exit(1);6019  } else if (DynamicRelrSize % DynamicRelrEntrySize) {6020    BC->errs() << "BOLT-ERROR: expected RELR table size to be divisible "6021               << "by RELR entry size\n";6022    exit(1);6023  }6024 6025  return Error::success();6026}6027 6028uint64_t RewriteInstance::getNewFunctionAddress(uint64_t OldAddress) {6029  const BinaryFunction *Function = BC->getBinaryFunctionAtAddress(OldAddress);6030  if (!Function)6031    return 0;6032 6033  return Function->getOutputAddress();6034}6035 6036uint64_t RewriteInstance::getNewFunctionOrDataAddress(uint64_t OldAddress) {6037  if (uint64_t Function = getNewFunctionAddress(OldAddress))6038    return Function;6039 6040  const BinaryData *BD = BC->getBinaryDataAtAddress(OldAddress);6041  if (BD && BD->isMoved())6042    return BD->getOutputAddress();6043 6044  if (const BinaryFunction *BF =6045          BC->getBinaryFunctionContainingAddress(OldAddress)) {6046    if (BF->isEmitted()) {6047      // If OldAddress is the another entry point of6048      // the function, then BOLT could get the new address.6049      if (BF->isMultiEntry()) {6050        for (const BinaryBasicBlock &BB : *BF)6051          if (BB.isEntryPoint() &&6052              (BF->getAddress() + BB.getOffset()) == OldAddress)6053            return BB.getOutputStartAddress();6054      }6055      BC->errs() << "BOLT-ERROR: unable to get new address corresponding to "6056                    "input address 0x"6057                 << Twine::utohexstr(OldAddress) << " in function " << *BF6058                 << ". Consider adding this function to --skip-funcs=...\n";6059      exit(1);6060    }6061  }6062 6063  return 0;6064}6065 6066void RewriteInstance::rewriteFile() {6067  std::error_code EC;6068  Out = std::make_unique<ToolOutputFile>(opts::OutputFilename, EC,6069                                         sys::fs::OF_None);6070  check_error(EC, "cannot create output executable file");6071 6072  raw_fd_ostream &OS = Out->os();6073 6074  // Copy allocatable part of the input.6075  OS << InputFile->getData().substr(0, FirstNonAllocatableOffset);6076 6077  auto Streamer = BC->createStreamer(OS);6078  // Make sure output stream has enough reserved space, otherwise6079  // pwrite() will fail.6080  uint64_t Offset = std::max(getFileOffsetForAddress(NextAvailableAddress),6081                             FirstNonAllocatableOffset);6082  Offset = OS.seek(Offset);6083  assert((Offset != (uint64_t)-1) && "Error resizing output file");6084 6085  // Overwrite functions with fixed output address. This is mostly used by6086  // non-relocation mode, with one exception: injected functions are covered6087  // here in both modes.6088  uint64_t CountOverwrittenFunctions = 0;6089  uint64_t OverwrittenScore = 0;6090  for (BinaryFunction *Function : BC->getAllBinaryFunctions()) {6091    if (Function->getImageAddress() == 0 || Function->getImageSize() == 0)6092      continue;6093 6094    assert(Function->getImageSize() <= Function->getMaxSize() &&6095           "Unexpected large function");6096 6097    const auto HasAddress = [](const FunctionFragment &FF) {6098      return FF.empty() ||6099             (FF.getImageAddress() != 0 && FF.getImageSize() != 0);6100    };6101    const bool SplitFragmentsHaveAddress =6102        llvm::all_of(Function->getLayout().getSplitFragments(), HasAddress);6103    if (Function->isSplit() && !SplitFragmentsHaveAddress) {6104      const auto HasNoAddress = [](const FunctionFragment &FF) {6105        return FF.getImageAddress() == 0 && FF.getImageSize() == 0;6106      };6107      assert(llvm::all_of(Function->getLayout().getSplitFragments(),6108                          HasNoAddress) &&6109             "Some split fragments have an address while others do not");6110      (void)HasNoAddress;6111      continue;6112    }6113 6114    OverwrittenScore += Function->getFunctionScore();6115    ++CountOverwrittenFunctions;6116 6117    // Overwrite function in the output file.6118    if (opts::Verbosity >= 2)6119      BC->outs() << "BOLT: rewriting function \"" << *Function << "\"\n";6120 6121    OS.pwrite(reinterpret_cast<char *>(Function->getImageAddress()),6122              Function->getImageSize(), Function->getFileOffset());6123 6124    // Write nops at the end of the function.6125    if (Function->getMaxSize() != std::numeric_limits<uint64_t>::max()) {6126      uint64_t Pos = OS.tell();6127      OS.seek(Function->getFileOffset() + Function->getImageSize());6128      BC->MAB->writeNopData(6129          OS, Function->getMaxSize() - Function->getImageSize(), &*BC->STI);6130 6131      OS.seek(Pos);6132    }6133 6134    if (!Function->isSplit())6135      continue;6136 6137    // Write cold part6138    if (opts::Verbosity >= 2) {6139      BC->outs() << formatv("BOLT: rewriting function \"{0}\" (split parts)\n",6140                            *Function);6141    }6142 6143    for (const FunctionFragment &FF :6144         Function->getLayout().getSplitFragments()) {6145      OS.pwrite(reinterpret_cast<char *>(FF.getImageAddress()),6146                FF.getImageSize(), FF.getFileOffset());6147    }6148  }6149 6150  // Print function statistics for non-relocation mode.6151  if (!BC->HasRelocations) {6152    BC->outs() << "BOLT: " << CountOverwrittenFunctions << " out of "6153               << BC->getBinaryFunctions().size()6154               << " functions were overwritten.\n";6155    if (BC->TotalScore != 0) {6156      double Coverage = OverwrittenScore / (double)BC->TotalScore * 100.0;6157      BC->outs() << format("BOLT-INFO: rewritten functions cover %.2lf",6158                           Coverage)6159                 << "% of the execution count of simple functions of "6160                    "this binary\n";6161    }6162  }6163 6164  if (BC->HasRelocations && opts::TrapOldCode) {6165    uint64_t SavedPos = OS.tell();6166    // Overwrite function body to make sure we never execute these instructions.6167    for (auto &BFI : BC->getBinaryFunctions()) {6168      BinaryFunction &BF = BFI.second;6169      if (!BF.getFileOffset() || !BF.isEmitted())6170        continue;6171      OS.seek(BF.getFileOffset());6172      StringRef TrapInstr = BC->MIB->getTrapFillValue();6173      unsigned NInstr = BF.getMaxSize() / TrapInstr.size();6174      for (unsigned I = 0; I < NInstr; ++I)6175        OS.write(TrapInstr.data(), TrapInstr.size());6176    }6177    OS.seek(SavedPos);6178  }6179 6180  // Write all allocatable sections - reloc-mode text is written here as well6181  for (BinarySection &Section : BC->allocatableSections()) {6182    if (!Section.isFinalized() || !Section.getOutputData()) {6183      LLVM_DEBUG(if (opts::Verbosity > 1) {6184        dbgs() << "BOLT-INFO: new section is finalized or !getOutputData, skip "6185               << Section.getName() << '\n';6186      });6187      continue;6188    }6189    if (Section.isLinkOnly()) {6190      LLVM_DEBUG(if (opts::Verbosity > 1) {6191        dbgs() << "BOLT-INFO: new section is link only, skip "6192               << Section.getName() << '\n';6193      });6194      continue;6195    }6196 6197    if (opts::Verbosity >= 1)6198      BC->outs() << "BOLT: writing new section " << Section.getName()6199                 << "\n data at 0x"6200                 << Twine::utohexstr(Section.getAllocAddress()) << "\n of size "6201                 << Section.getOutputSize() << "\n at offset "6202                 << Section.getOutputFileOffset() << " with content size "6203                 << Section.getOutputContents().size() << '\n';6204    OS.seek(Section.getOutputFileOffset());6205    Section.write(OS);6206  }6207 6208  for (BinarySection &Section : BC->allocatableSections())6209    Section.flushPendingRelocations(OS, [this](const MCSymbol *S) {6210      return getNewValueForSymbol(S->getName());6211    });6212 6213  // If .eh_frame is present create .eh_frame_hdr.6214  if (EHFrameSection)6215    writeEHFrameHeader();6216 6217  // Add BOLT Addresses Translation maps to allow profile collection to6218  // happen in the output binary6219  if (opts::EnableBAT)6220    addBATSection();6221 6222  // Patch program header table.6223  if (!BC->IsLinuxKernel) {6224    updateSegmentInfo();6225    patchELFPHDRTable();6226  }6227 6228  // Finalize memory image of section string table.6229  finalizeSectionStringTable();6230 6231  // Update symbol tables.6232  patchELFSymTabs();6233 6234  if (opts::EnableBAT)6235    encodeBATSection();6236 6237  // Copy non-allocatable sections once allocatable part is finished.6238  rewriteNoteSections();6239 6240  if (BC->HasRelocations) {6241    patchELFAllocatableRelaSections();6242    patchELFAllocatableRelrSection();6243    patchELFGOT();6244  }6245 6246  // Patch dynamic section/segment.6247  patchELFDynamic();6248 6249  // Update ELF book-keeping info.6250  patchELFSectionHeaderTable();6251 6252  if (opts::PrintSections) {6253    BC->outs() << "BOLT-INFO: Sections after processing:\n";6254    BC->printSections(BC->outs());6255  }6256 6257  Out->keep();6258  EC = sys::fs::setPermissions(6259      opts::OutputFilename,6260      static_cast<sys::fs::perms>(sys::fs::perms::all_all &6261                                  ~sys::fs::getUmask()));6262  check_error(EC, "cannot set permissions of output file");6263}6264 6265void RewriteInstance::writeEHFrameHeader() {6266  BinarySection *NewEHFrameSection =6267      getSection(getNewSecPrefix() + getEHFrameSectionName());6268 6269  // No need to update the header if no new .eh_frame was created.6270  if (!NewEHFrameSection)6271    return;6272 6273  DWARFDebugFrame NewEHFrame(BC->TheTriple->getArch(), true,6274                             NewEHFrameSection->getOutputAddress());6275  Error E = NewEHFrame.parse(DWARFDataExtractor(6276      NewEHFrameSection->getOutputContents(), BC->AsmInfo->isLittleEndian(),6277      BC->AsmInfo->getCodePointerSize()));6278  check_error(std::move(E), "failed to parse EH frame");6279 6280  uint64_t RelocatedEHFrameAddress = 0;6281  StringRef RelocatedEHFrameContents;6282  BinarySection *RelocatedEHFrameSection =6283      getSection(".relocated" + getEHFrameSectionName());6284  if (RelocatedEHFrameSection) {6285    RelocatedEHFrameAddress = RelocatedEHFrameSection->getOutputAddress();6286    RelocatedEHFrameContents = RelocatedEHFrameSection->getOutputContents();6287  }6288  DWARFDebugFrame RelocatedEHFrame(BC->TheTriple->getArch(), true,6289                                   RelocatedEHFrameAddress);6290  Error Er = RelocatedEHFrame.parse(DWARFDataExtractor(6291      RelocatedEHFrameContents, BC->AsmInfo->isLittleEndian(),6292      BC->AsmInfo->getCodePointerSize()));6293  check_error(std::move(Er), "failed to parse EH frame");6294 6295  LLVM_DEBUG(dbgs() << "BOLT: writing a new " << getEHFrameHdrSectionName()6296                    << '\n');6297 6298  // Try to overwrite the original .eh_frame_hdr if the size permits.6299  uint64_t EHFrameHdrOutputAddress = 0;6300  uint64_t EHFrameHdrFileOffset = 0;6301  std::vector<char> NewEHFrameHdr;6302  BinarySection *OldEHFrameHdrSection = getSection(getEHFrameHdrSectionName());6303  if (OldEHFrameHdrSection) {6304    NewEHFrameHdr = CFIRdWrt->generateEHFrameHeader(6305        RelocatedEHFrame, NewEHFrame, OldEHFrameHdrSection->getAddress());6306    if (NewEHFrameHdr.size() <= OldEHFrameHdrSection->getSize()) {6307      BC->outs() << "BOLT-INFO: rewriting " << getEHFrameHdrSectionName()6308                 << " in-place\n";6309      EHFrameHdrOutputAddress = OldEHFrameHdrSection->getAddress();6310      EHFrameHdrFileOffset = OldEHFrameHdrSection->getInputFileOffset();6311    } else {6312      OldEHFrameHdrSection->setOutputName(getOrgSecPrefix() +6313                                          getEHFrameHdrSectionName());6314      OldEHFrameHdrSection = nullptr;6315    }6316  }6317 6318  // If there was not enough space, allocate more memory for .eh_frame_hdr.6319  if (!OldEHFrameHdrSection) {6320    NextAvailableAddress =6321        appendPadding(Out->os(), NextAvailableAddress, EHFrameHdrAlign);6322 6323    EHFrameHdrOutputAddress = NextAvailableAddress;6324    EHFrameHdrFileOffset = getFileOffsetForAddress(NextAvailableAddress);6325 6326    NewEHFrameHdr = CFIRdWrt->generateEHFrameHeader(6327        RelocatedEHFrame, NewEHFrame, EHFrameHdrOutputAddress);6328 6329    NextAvailableAddress += NewEHFrameHdr.size();6330    if (!BC->BOLTReserved.empty() &&6331        (NextAvailableAddress > BC->BOLTReserved.end())) {6332      BC->errs() << "BOLT-ERROR: unable to fit " << getEHFrameHdrSectionName()6333                 << " into reserved space\n";6334      exit(1);6335    }6336 6337    // Create a new entry in the section header table.6338    const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/true,6339                                                   /*IsText=*/false,6340                                                   /*IsAllocatable=*/true);6341    BinarySection &EHFrameHdrSec = BC->registerOrUpdateSection(6342        getNewSecPrefix() + getEHFrameHdrSectionName(), ELF::SHT_PROGBITS,6343        Flags, nullptr, NewEHFrameHdr.size(), /*Alignment=*/1);6344    EHFrameHdrSec.setOutputFileOffset(EHFrameHdrFileOffset);6345    EHFrameHdrSec.setOutputAddress(EHFrameHdrOutputAddress);6346    EHFrameHdrSec.setOutputName(getEHFrameHdrSectionName());6347  }6348 6349  Out->os().seek(EHFrameHdrFileOffset);6350  Out->os().write(NewEHFrameHdr.data(), NewEHFrameHdr.size());6351 6352  // Pad the contents if overwriting in-place.6353  if (OldEHFrameHdrSection)6354    Out->os().write_zeros(OldEHFrameHdrSection->getSize() -6355                          NewEHFrameHdr.size());6356 6357  // Merge new .eh_frame with the relocated original so that gdb can locate all6358  // FDEs.6359  if (RelocatedEHFrameSection) {6360    const uint64_t NewEHFrameSectionSize =6361        RelocatedEHFrameSection->getOutputAddress() +6362        RelocatedEHFrameSection->getOutputSize() -6363        NewEHFrameSection->getOutputAddress();6364    NewEHFrameSection->updateContents(NewEHFrameSection->getOutputData(),6365                                      NewEHFrameSectionSize);6366    BC->deregisterSection(*RelocatedEHFrameSection);6367  }6368 6369  LLVM_DEBUG(dbgs() << "BOLT-DEBUG: size of .eh_frame after merge is "6370                    << NewEHFrameSection->getOutputSize() << '\n');6371}6372 6373uint64_t RewriteInstance::getNewValueForSymbol(const StringRef Name) {6374  auto Value = Linker->lookupSymbolInfo(Name);6375  if (Value)6376    return Value->Address;6377 6378  // Return the original value if we haven't emitted the symbol.6379  BinaryData *BD = BC->getBinaryDataByName(Name);6380  if (!BD)6381    return 0;6382 6383  return BD->getAddress();6384}6385 6386uint64_t RewriteInstance::getFileOffsetForAddress(uint64_t Address) const {6387  // Check if it's possibly part of the new segment.6388  if (NewTextSegmentAddress && Address >= NewTextSegmentAddress)6389    return Address - NewTextSegmentAddress + NewTextSegmentOffset;6390 6391  // Find an existing segment that matches the address.6392  const auto SegmentInfoI = BC->SegmentMapInfo.upper_bound(Address);6393  if (SegmentInfoI == BC->SegmentMapInfo.begin())6394    return 0;6395 6396  const SegmentInfo &SegmentInfo = std::prev(SegmentInfoI)->second;6397  if (Address < SegmentInfo.Address ||6398      Address >= SegmentInfo.Address + SegmentInfo.FileSize)6399    return 0;6400 6401  return SegmentInfo.FileOffset + Address - SegmentInfo.Address;6402}6403 6404bool RewriteInstance::willOverwriteSection(StringRef SectionName) {6405  if (llvm::is_contained(SectionsToOverwrite, SectionName))6406    return true;6407  if (llvm::is_contained(DebugSectionsToOverwrite, SectionName))6408    return true;6409 6410  ErrorOr<BinarySection &> Section = BC->getUniqueSectionByName(SectionName);6411  return Section && Section->isAllocatable() && Section->isFinalized();6412}6413 6414bool RewriteInstance::isDebugSection(StringRef SectionName) {6415  if (SectionName.starts_with(".debug_") ||6416      SectionName.starts_with(".zdebug_") || SectionName == ".gdb_index" ||6417      SectionName == ".stab" || SectionName == ".stabstr")6418    return true;6419 6420  return false;6421}6422