1254 lines · cpp
1//===- bolt/Core/BinaryEmitter.cpp - Emit code and data -------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the collection of functions and classes used for10// emission of code and data into object/binary file.11//12//===----------------------------------------------------------------------===//13 14#include "bolt/Core/BinaryEmitter.h"15#include "bolt/Core/BinaryContext.h"16#include "bolt/Core/BinaryFunction.h"17#include "bolt/Core/DebugData.h"18#include "bolt/Core/FunctionLayout.h"19#include "bolt/Utils/CommandLineOpts.h"20#include "bolt/Utils/Utils.h"21#include "llvm/DebugInfo/DWARF/DWARFCompileUnit.h"22#include "llvm/MC/MCSection.h"23#include "llvm/MC/MCStreamer.h"24#include "llvm/Support/CommandLine.h"25#include "llvm/Support/LEB128.h"26#include "llvm/Support/SMLoc.h"27 28#define DEBUG_TYPE "bolt"29 30using namespace llvm;31using namespace bolt;32 33namespace opts {34 35extern cl::opt<JumpTableSupportLevel> JumpTables;36extern cl::opt<bool> PreserveBlocksAlignment;37 38cl::opt<bool> AlignBlocks("align-blocks", cl::desc("align basic blocks"),39 cl::cat(BoltOptCategory));40 41static cl::list<std::string>42BreakFunctionNames("break-funcs",43 cl::CommaSeparated,44 cl::desc("list of functions to core dump on (debugging)"),45 cl::value_desc("func1,func2,func3,..."),46 cl::Hidden,47 cl::cat(BoltCategory));48 49static cl::list<std::string>50 FunctionPadSpec("pad-funcs", cl::CommaSeparated,51 cl::desc("list of functions to pad with amount of bytes"),52 cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."),53 cl::Hidden, cl::cat(BoltCategory));54 55static cl::list<std::string> FunctionPadBeforeSpec(56 "pad-funcs-before", cl::CommaSeparated,57 cl::desc("list of functions to pad with amount of bytes"),58 cl::value_desc("func1:pad1,func2:pad2,func3:pad3,..."), cl::Hidden,59 cl::cat(BoltCategory));60 61static cl::opt<bool> MarkFuncs(62 "mark-funcs",63 cl::desc("mark function boundaries with break instruction to make "64 "sure we accidentally don't cross them"),65 cl::ReallyHidden, cl::cat(BoltCategory));66 67static cl::opt<bool> PrintJumpTables("print-jump-tables",68 cl::desc("print jump tables"), cl::Hidden,69 cl::cat(BoltCategory));70 71static cl::opt<bool>72X86AlignBranchBoundaryHotOnly("x86-align-branch-boundary-hot-only",73 cl::desc("only apply branch boundary alignment in hot code"),74 cl::init(true),75 cl::cat(BoltOptCategory));76 77size_t padFunction(std::map<std::string, size_t> &FunctionPadding,78 const cl::list<std::string> &Spec,79 const BinaryFunction &Function) {80 if (FunctionPadding.empty() && !Spec.empty()) {81 for (const std::string &Spec : Spec) {82 size_t N = Spec.find(':');83 if (N == std::string::npos)84 continue;85 std::string Name = Spec.substr(0, N);86 size_t Padding = std::stoull(Spec.substr(N + 1));87 FunctionPadding[Name] = Padding;88 }89 }90 91 for (auto &FPI : FunctionPadding) {92 std::string Name = FPI.first;93 size_t Padding = FPI.second;94 if (Function.hasNameRegex(Name))95 return Padding;96 }97 98 return 0;99}100 101size_t padFunctionBefore(const BinaryFunction &Function) {102 static std::map<std::string, size_t> CacheFunctionPadding;103 return padFunction(CacheFunctionPadding, FunctionPadBeforeSpec, Function);104}105size_t padFunctionAfter(const BinaryFunction &Function) {106 static std::map<std::string, size_t> CacheFunctionPadding;107 return padFunction(CacheFunctionPadding, FunctionPadSpec, Function);108}109 110} // namespace opts111 112namespace {113using JumpTable = bolt::JumpTable;114 115class BinaryEmitter {116private:117 BinaryEmitter(const BinaryEmitter &) = delete;118 BinaryEmitter &operator=(const BinaryEmitter &) = delete;119 120 MCStreamer &Streamer;121 BinaryContext &BC;122 123public:124 BinaryEmitter(MCStreamer &Streamer, BinaryContext &BC)125 : Streamer(Streamer), BC(BC) {}126 127 /// Emit all code and data.128 void emitAll(StringRef OrgSecPrefix);129 130 /// Emit function code. The caller is responsible for emitting function131 /// symbol(s) and setting the section to emit the code to.132 void emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,133 bool EmitCodeOnly = false);134 135private:136 /// Emit function code.137 void emitFunctions();138 139 /// Emit a single function.140 bool emitFunction(BinaryFunction &BF, FunctionFragment &FF);141 142 /// Helper for emitFunctionBody to write data inside a function143 /// (used for AArch64)144 void emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,145 BinaryFunction *OnBehalfOf = nullptr);146 147 /// Emit jump tables for the function.148 void emitJumpTables(const BinaryFunction &BF);149 150 /// Emit jump table data. Callee supplies sections for the data.151 void emitJumpTable(const JumpTable &JT, MCSection *HotSection,152 MCSection *ColdSection);153 154 void emitCFIInstruction(const MCCFIInstruction &Inst) const;155 156 /// Emit exception handling ranges for the function fragment.157 void emitLSDA(BinaryFunction &BF, const FunctionFragment &FF);158 159 /// Emit line number information corresponding to \p NewLoc. \p PrevLoc160 /// provides a context for de-duplication of line number info.161 /// \p FirstInstr indicates if \p NewLoc represents the first instruction162 /// in a sequence, such as a function fragment.163 ///164 /// If \p NewLoc location matches \p PrevLoc, no new line number entry will be165 /// created and the function will return \p PrevLoc while \p InstrLabel will166 /// be ignored. Otherwise, the caller should use \p InstrLabel to mark the167 /// corresponding instruction by emitting \p InstrLabel before it.168 /// If \p InstrLabel is set by the caller, its value will be used with \p169 /// \p NewLoc. If it was nullptr on entry, it will be populated with a pointer170 /// to a new temp symbol used with \p NewLoc.171 ///172 /// Return new current location which is either \p NewLoc or \p PrevLoc.173 SMLoc emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc, SMLoc PrevLoc,174 bool FirstInstr, MCSymbol *&InstrLabel);175 176 /// Use \p FunctionEndSymbol to mark the end of the line info sequence.177 /// Note that it does not automatically result in the insertion of the EOS178 /// marker in the line table program, but provides one to the DWARF generator179 /// when it needs it.180 void emitLineInfoEnd(const BinaryFunction &BF, MCSymbol *FunctionEndSymbol,181 const DWARFUnit &Unit);182 183 /// Emit debug line info for unprocessed functions from CUs that include184 /// emitted functions.185 void emitDebugLineInfoForOriginalFunctions();186 187 /// Emit debug line for CUs that were not modified.188 void emitDebugLineInfoForUnprocessedCUs();189 190 /// Emit data sections that have code references in them.191 void emitDataSections(StringRef OrgSecPrefix);192};193 194} // anonymous namespace195 196void BinaryEmitter::emitAll(StringRef OrgSecPrefix) {197 Streamer.initSections(false, *BC.STI);198 Streamer.setUseAssemblerInfoForParsing(false);199 200 if (opts::UpdateDebugSections && BC.isELF()) {201 // Force the emission of debug line info into allocatable section to ensure202 // JITLink will process it.203 //204 // NB: on MachO all sections are required for execution, hence no need205 // to change flags/attributes.206 MCSectionELF *ELFDwarfLineSection =207 static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineSection());208 ELFDwarfLineSection->setFlags(ELF::SHF_ALLOC);209 MCSectionELF *ELFDwarfLineStrSection =210 static_cast<MCSectionELF *>(BC.MOFI->getDwarfLineStrSection());211 ELFDwarfLineStrSection->setFlags(ELF::SHF_ALLOC);212 }213 214 if (RuntimeLibrary *RtLibrary = BC.getRuntimeLibrary())215 RtLibrary->emitBinary(BC, Streamer);216 217 BC.getTextSection()->setAlignment(Align(opts::AlignText));218 219 emitFunctions();220 221 if (opts::UpdateDebugSections) {222 emitDebugLineInfoForOriginalFunctions();223 DwarfLineTable::emit(BC, Streamer);224 }225 226 emitDataSections(OrgSecPrefix);227 228 // TODO Enable for Mach-O once BinaryContext::getDataSection supports it.229 if (BC.isELF())230 AddressMap::emit(Streamer, BC);231 Streamer.setUseAssemblerInfoForParsing(true);232}233 234void BinaryEmitter::emitFunctions() {235 auto emit = [&](const std::vector<BinaryFunction *> &Functions) {236 const bool HasProfile = BC.NumProfiledFuncs > 0;237 const bool OriginalAllowAutoPadding = Streamer.getAllowAutoPadding();238 for (BinaryFunction *Function : Functions) {239 if (!BC.shouldEmit(*Function))240 continue;241 242 LLVM_DEBUG(dbgs() << "BOLT: generating code for function \"" << *Function243 << "\" : " << Function->getFunctionNumber() << '\n');244 245 // Was any part of the function emitted.246 bool Emitted = false;247 248 // Turn off Intel JCC Erratum mitigation for cold code if requested249 if (HasProfile && opts::X86AlignBranchBoundaryHotOnly &&250 !Function->hasValidProfile())251 Streamer.setAllowAutoPadding(false);252 253 FunctionLayout &Layout = Function->getLayout();254 Emitted |= emitFunction(*Function, Layout.getMainFragment());255 256 if (Function->isSplit()) {257 if (opts::X86AlignBranchBoundaryHotOnly)258 Streamer.setAllowAutoPadding(false);259 260 assert((Layout.fragment_size() == 1 || Function->isSimple()) &&261 "Only simple functions can have fragments");262 for (FunctionFragment &FF : Layout.getSplitFragments()) {263 // Skip empty fragments so no symbols and sections for empty fragments264 // are generated265 if (FF.empty() && !Function->hasConstantIsland())266 continue;267 Emitted |= emitFunction(*Function, FF);268 }269 }270 271 Streamer.setAllowAutoPadding(OriginalAllowAutoPadding);272 273 if (Emitted)274 Function->setEmitted(/*KeepCFG=*/opts::PrintCacheMetrics);275 }276 };277 278 // Mark the start of hot text.279 if (opts::HotText) {280 Streamer.switchSection(BC.getTextSection());281 Streamer.emitLabel(BC.getHotTextStartSymbol());282 }283 284 // Emit functions in sorted order.285 std::vector<BinaryFunction *> SortedFunctions = BC.getSortedFunctions();286 emit(SortedFunctions);287 288 // Emit functions added by BOLT.289 emit(BC.getInjectedBinaryFunctions());290 291 // Mark the end of hot text.292 if (opts::HotText) {293 if (BC.HasWarmSection)294 Streamer.switchSection(BC.getCodeSection(BC.getWarmCodeSectionName()));295 else296 Streamer.switchSection(BC.getTextSection());297 Streamer.emitLabel(BC.getHotTextEndSymbol());298 }299}300 301bool BinaryEmitter::emitFunction(BinaryFunction &Function,302 FunctionFragment &FF) {303 if (Function.size() == 0 && !Function.hasIslandsInfo())304 return false;305 306 if (Function.getState() == BinaryFunction::State::Empty)307 return false;308 309 // Avoid emitting function without instructions when overwriting the original310 // function in-place. Otherwise, emit the empty function to define the symbol.311 if (!BC.HasRelocations && !Function.hasNonPseudoInstructions())312 return false;313 314 MCSection *Section =315 BC.getCodeSection(Function.getCodeSectionName(FF.getFragmentNum()));316 Streamer.switchSection(Section);317 Section->setHasInstructions(true);318 BC.Ctx->addGenDwarfSection(Section);319 320 if (BC.HasRelocations) {321 // Set section alignment to at least maximum possible object alignment.322 // We need this to support LongJmp and other passes that calculates323 // tentative layout.324 Section->ensureMinAlignment(Align(opts::AlignFunctions));325 326 Streamer.emitCodeAlignment(Function.getMinAlign(), &*BC.STI);327 uint16_t MaxAlignBytes = FF.isSplitFragment()328 ? Function.getMaxColdAlignmentBytes()329 : Function.getMaxAlignmentBytes();330 if (MaxAlignBytes > 0)331 Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI, MaxAlignBytes);332 } else {333 Streamer.emitCodeAlignment(Function.getAlign(), &*BC.STI);334 }335 336 if (size_t Padding = opts::padFunctionBefore(Function)) {337 // Handle padFuncsBefore after the above alignment logic but before338 // symbol addresses are decided.339 if (!BC.HasRelocations) {340 BC.errs() << "BOLT-ERROR: -pad-before-funcs is not supported in "341 << "non-relocation mode\n";342 exit(1);343 }344 345 // Preserve Function.getMinAlign().346 if (!isAligned(Function.getMinAlign(), Padding)) {347 BC.errs() << "BOLT-ERROR: user-requested " << Padding348 << " padding bytes before function " << Function349 << " is not a multiple of the minimum function alignment ("350 << Function.getMinAlign().value() << ").\n";351 exit(1);352 }353 354 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding before function " << Function355 << " with " << Padding << " bytes\n");356 357 // Since the padding is not executed, it can be null bytes.358 Streamer.emitFill(Padding, 0);359 }360 361 MCContext &Context = Streamer.getContext();362 const MCAsmInfo *MAI = Context.getAsmInfo();363 364 MCSymbol *const StartSymbol = Function.getSymbol(FF.getFragmentNum());365 366 // Emit all symbols associated with the main function entry.367 if (FF.isMainFragment()) {368 for (MCSymbol *Symbol : Function.getSymbols()) {369 Streamer.emitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);370 Streamer.emitLabel(Symbol);371 }372 } else {373 Streamer.emitSymbolAttribute(StartSymbol, MCSA_ELF_TypeFunction);374 Streamer.emitLabel(StartSymbol);375 }376 377 const bool NeedsFDE =378 Function.hasCFI() && !(Function.isPatch() && Function.isAnonymous());379 // Emit CFI start380 if (NeedsFDE) {381 Streamer.emitCFIStartProc(/*IsSimple=*/false);382 if (Function.getPersonalityFunction() != nullptr)383 Streamer.emitCFIPersonality(Function.getPersonalityFunction(),384 Function.getPersonalityEncoding());385 MCSymbol *LSDASymbol = Function.getLSDASymbol(FF.getFragmentNum());386 if (LSDASymbol)387 Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding);388 else389 Streamer.emitCFILsda(0, dwarf::DW_EH_PE_omit);390 // Emit CFI instructions relative to the CIE391 for (const MCCFIInstruction &CFIInstr : Function.cie()) {392 // Only write CIE CFI insns that LLVM will not already emit393 const std::vector<MCCFIInstruction> &FrameInstrs =394 MAI->getInitialFrameState();395 if (!llvm::is_contained(FrameInstrs, CFIInstr))396 emitCFIInstruction(CFIInstr);397 }398 }399 400 assert((Function.empty() || !(*Function.begin()).isCold()) &&401 "first basic block should never be cold");402 403 // Emit UD2 at the beginning if requested by user.404 if (!opts::BreakFunctionNames.empty()) {405 for (std::string &Name : opts::BreakFunctionNames) {406 if (Function.hasNameRegex(Name)) {407 Streamer.emitIntValue(0x0B0F, 2); // UD2: 0F 0B408 break;409 }410 }411 }412 413 // Emit code.414 emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);415 416 // Emit padding if requested.417 if (size_t Padding = opts::padFunctionAfter(Function)) {418 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: padding function " << Function << " with "419 << Padding << " bytes\n");420 Streamer.emitFill(Padding, MAI->getTextAlignFillValue());421 }422 423 if (opts::MarkFuncs)424 Streamer.emitBytes(BC.MIB->getTrapFillValue());425 426 // Emit CFI end427 if (NeedsFDE)428 Streamer.emitCFIEndProc();429 430 MCSymbol *EndSymbol = Function.getFunctionEndLabel(FF.getFragmentNum());431 Streamer.emitLabel(EndSymbol);432 433 if (MAI->hasDotTypeDotSizeDirective()) {434 const MCExpr *SizeExpr = MCBinaryExpr::createSub(435 MCSymbolRefExpr::create(EndSymbol, Context),436 MCSymbolRefExpr::create(StartSymbol, Context), Context);437 Streamer.emitELFSize(StartSymbol, SizeExpr);438 }439 440 if (opts::UpdateDebugSections && !Function.getDWARFUnits().empty())441 for (const auto &[_, Unit] : Function.getDWARFUnits())442 emitLineInfoEnd(Function, EndSymbol, *Unit);443 444 // Exception handling info for the function.445 emitLSDA(Function, FF);446 447 if (FF.isMainFragment() && opts::JumpTables > JTS_NONE)448 emitJumpTables(Function);449 450 return true;451}452 453void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,454 bool EmitCodeOnly) {455 if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) {456 assert(BF.getLayout().isHotColdSplit() &&457 "Constant island support only with hot/cold split");458 BF.duplicateConstantIslands();459 }460 461 // Track the first emitted instruction with debug info.462 bool FirstInstr = true;463 for (BinaryBasicBlock *const BB : FF) {464 if ((opts::AlignBlocks || opts::PreserveBlocksAlignment) &&465 BB->getAlignment() > 1)466 Streamer.emitCodeAlignment(BB->getAlign(), &*BC.STI,467 BB->getAlignmentMaxBytes());468 Streamer.emitLabel(BB->getLabel());469 if (!EmitCodeOnly) {470 if (MCSymbol *EntrySymbol = BF.getSecondaryEntryPointSymbol(*BB))471 Streamer.emitLabel(EntrySymbol);472 }473 474 SMLoc LastLocSeen;475 for (auto I = BB->begin(), E = BB->end(); I != E; ++I) {476 MCInst &Instr = *I;477 478 if (EmitCodeOnly && BC.MIB->isPseudo(Instr))479 continue;480 481 // Handle pseudo instructions.482 if (BC.MIB->isCFI(Instr)) {483 emitCFIInstruction(*BF.getCFIFor(Instr));484 continue;485 }486 487 if (!EmitCodeOnly) {488 // A symbol to be emitted before the instruction to mark its location.489 MCSymbol *InstrLabel = BC.MIB->getInstLabel(Instr);490 491 if (opts::UpdateDebugSections && !BF.getDWARFUnits().empty()) {492 LastLocSeen = emitLineInfo(BF, Instr.getLoc(), LastLocSeen,493 FirstInstr, InstrLabel);494 FirstInstr = false;495 }496 497 // Prepare to tag this location with a label if we need to keep track of498 // the location of calls/returns for BOLT address translation maps499 if (BF.requiresAddressTranslation() && BC.MIB->getOffset(Instr)) {500 const uint32_t Offset = *BC.MIB->getOffset(Instr);501 if (!InstrLabel)502 InstrLabel = BC.Ctx->createTempSymbol();503 BB->getLocSyms().emplace_back(Offset, InstrLabel);504 }505 506 if (InstrLabel)507 Streamer.emitLabel(InstrLabel);508 }509 510 // Emit sized NOPs via MCAsmBackend::writeNopData() interface on x86.511 // This is a workaround for invalid NOPs handling by asm/disasm layer.512 if (BC.isX86() && BC.MIB->isNoop(Instr)) {513 if (std::optional<uint32_t> Size = BC.MIB->getSize(Instr)) {514 SmallString<15> Code;515 raw_svector_ostream VecOS(Code);516 BC.MAB->writeNopData(VecOS, *Size, BC.STI.get());517 Streamer.emitBytes(Code);518 continue;519 }520 }521 522 Streamer.emitInstruction(Instr, *BC.STI);523 }524 }525 526 if (!EmitCodeOnly)527 emitConstantIslands(BF, FF.isSplitFragment());528}529 530void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,531 BinaryFunction *OnBehalfOf) {532 if (!BF.hasIslandsInfo())533 return;534 535 BinaryFunction::IslandInfo &Islands = BF.getIslandInfo();536 if (Islands.DataOffsets.empty() && Islands.Dependency.empty())537 return;538 539 // AArch64 requires CI to be aligned to 8 bytes due to access instructions540 // restrictions. E.g. the ldr with imm, where imm must be aligned to 8 bytes.541 const uint16_t Alignment = OnBehalfOf542 ? OnBehalfOf->getConstantIslandAlignment()543 : BF.getConstantIslandAlignment();544 Streamer.emitCodeAlignment(Align(Alignment), &*BC.STI);545 546 if (!OnBehalfOf) {547 if (!EmitColdPart)548 Streamer.emitLabel(BF.getFunctionConstantIslandLabel());549 else550 Streamer.emitLabel(BF.getFunctionColdConstantIslandLabel());551 }552 553 assert((!OnBehalfOf || Islands.Proxies[OnBehalfOf].size() > 0) &&554 "spurious OnBehalfOf constant island emission");555 556 assert(!BF.isInjected() &&557 "injected functions should not have constant islands");558 // Raw contents of the function.559 StringRef SectionContents = BF.getOriginSection()->getContents();560 561 // Raw contents of the function.562 StringRef FunctionContents = SectionContents.substr(563 BF.getAddress() - BF.getOriginSection()->getAddress(), BF.getMaxSize());564 565 if (opts::Verbosity && !OnBehalfOf)566 BC.outs() << "BOLT-INFO: emitting constant island for function " << BF567 << "\n";568 569 // We split the island into smaller blocks and output labels between them.570 auto IS = Islands.Offsets.begin();571 for (auto DataIter = Islands.DataOffsets.begin();572 DataIter != Islands.DataOffsets.end(); ++DataIter) {573 uint64_t FunctionOffset = *DataIter;574 uint64_t EndOffset = 0ULL;575 576 // Determine size of this data chunk577 auto NextData = std::next(DataIter);578 auto CodeIter = Islands.CodeOffsets.lower_bound(*DataIter);579 if (CodeIter == Islands.CodeOffsets.end() &&580 NextData == Islands.DataOffsets.end())581 EndOffset = BF.getMaxSize();582 else if (CodeIter == Islands.CodeOffsets.end())583 EndOffset = *NextData;584 else if (NextData == Islands.DataOffsets.end())585 EndOffset = *CodeIter;586 else587 EndOffset = (*CodeIter > *NextData) ? *NextData : *CodeIter;588 589 if (FunctionOffset == EndOffset)590 continue; // Size is zero, nothing to emit591 592 auto emitCI = [&](uint64_t &FunctionOffset, uint64_t EndOffset) {593 if (FunctionOffset >= EndOffset)594 return;595 596 for (auto It = Islands.Relocations.lower_bound(FunctionOffset);597 It != Islands.Relocations.end(); ++It) {598 if (It->first >= EndOffset)599 break;600 601 const Relocation &Relocation = It->second;602 if (FunctionOffset < Relocation.Offset) {603 Streamer.emitBytes(604 FunctionContents.slice(FunctionOffset, Relocation.Offset));605 FunctionOffset = Relocation.Offset;606 }607 608 LLVM_DEBUG(609 dbgs() << "BOLT-DEBUG: emitting constant island relocation"610 << " for " << BF << " at offset 0x"611 << Twine::utohexstr(Relocation.Offset) << " with size "612 << Relocation::getSizeForType(Relocation.Type) << '\n');613 614 FunctionOffset += Relocation.emit(&Streamer);615 }616 617 assert(FunctionOffset <= EndOffset && "overflow error");618 if (FunctionOffset < EndOffset) {619 Streamer.emitBytes(FunctionContents.slice(FunctionOffset, EndOffset));620 FunctionOffset = EndOffset;621 }622 };623 624 // Emit labels, relocs and data625 while (IS != Islands.Offsets.end() && IS->first < EndOffset) {626 auto NextLabelOffset =627 IS == Islands.Offsets.end() ? EndOffset : IS->first;628 auto NextStop = std::min(NextLabelOffset, EndOffset);629 assert(NextStop <= EndOffset && "internal overflow error");630 emitCI(FunctionOffset, NextStop);631 if (IS != Islands.Offsets.end() && FunctionOffset == IS->first) {632 // This is a slightly complex code to decide which label to emit. We633 // have 4 cases to handle: regular symbol, cold symbol, regular or cold634 // symbol being emitted on behalf of an external function.635 if (!OnBehalfOf) {636 if (!EmitColdPart) {637 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "638 << IS->second->getName() << " at offset 0x"639 << Twine::utohexstr(IS->first) << '\n');640 if (IS->second->isUndefined())641 Streamer.emitLabel(IS->second);642 else643 assert(BF.hasName(std::string(IS->second->getName())));644 } else if (Islands.ColdSymbols.count(IS->second) != 0) {645 LLVM_DEBUG(dbgs()646 << "BOLT-DEBUG: emitted label "647 << Islands.ColdSymbols[IS->second]->getName() << '\n');648 if (Islands.ColdSymbols[IS->second]->isUndefined())649 Streamer.emitLabel(Islands.ColdSymbols[IS->second]);650 }651 } else {652 if (!EmitColdPart) {653 if (MCSymbol *Sym = Islands.Proxies[OnBehalfOf][IS->second]) {654 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label "655 << Sym->getName() << '\n');656 Streamer.emitLabel(Sym);657 }658 } else if (MCSymbol *Sym =659 Islands.ColdProxies[OnBehalfOf][IS->second]) {660 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitted label " << Sym->getName()661 << '\n');662 Streamer.emitLabel(Sym);663 }664 }665 ++IS;666 }667 }668 assert(FunctionOffset <= EndOffset && "overflow error");669 emitCI(FunctionOffset, EndOffset);670 }671 assert(IS == Islands.Offsets.end() && "some symbols were not emitted!");672 673 if (OnBehalfOf)674 return;675 // Now emit constant islands from other functions that we may have used in676 // this function.677 for (BinaryFunction *ExternalFunc : Islands.Dependency)678 emitConstantIslands(*ExternalFunc, EmitColdPart, &BF);679}680 681SMLoc BinaryEmitter::emitLineInfo(const BinaryFunction &BF, SMLoc NewLoc,682 SMLoc PrevLoc, bool FirstInstr,683 MCSymbol *&InstrLabel) {684 if (NewLoc.getPointer() == nullptr ||685 NewLoc.getPointer() == PrevLoc.getPointer())686 return PrevLoc;687 const ClusteredRows *Cluster = ClusteredRows::fromSMLoc(NewLoc);688 689 auto addToLineTable = [&](DebugLineTableRowRef RowReference,690 const DWARFUnit &TargetCU, unsigned Flags,691 MCSymbol &InstrLabel,692 const DWARFDebugLine::Row &CurrentRow) {693 const uint64_t TargetUnitIndex = TargetCU.getOffset();694 unsigned TargetFilenum = CurrentRow.File;695 const uint32_t CurrentUnitIndex = RowReference.DwCompileUnitIndex;696 // If the CU id from the current instruction location does not697 // match the target CU id, it means that we have come across some698 // inlined code (by BOLT). We must look up the CU for the instruction's699 // original function and get the line table from that.700 if (TargetUnitIndex != CurrentUnitIndex) {701 // Add filename from the inlined function to the current CU.702 TargetFilenum = BC.addDebugFilenameToUnit(703 TargetUnitIndex, CurrentUnitIndex, CurrentRow.File);704 }705 BC.Ctx->setCurrentDwarfLoc(TargetFilenum, CurrentRow.Line,706 CurrentRow.Column, Flags, CurrentRow.Isa,707 CurrentRow.Discriminator);708 const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();709 BC.Ctx->clearDwarfLocSeen();710 const MCLineSection::MCLineDivisionMap &MapLineEntries =711 BC.getDwarfLineTable(TargetUnitIndex)712 .getMCLineSections()713 .getMCLineEntries();714 const auto *It = MapLineEntries.find(Streamer.getCurrentSectionOnly());715 MCDwarfLineEntry NewLineEntry = MCDwarfLineEntry(&InstrLabel, DwarfLoc);716 717 // Check if line table exists and has entries before doing comparison.718 if (It != MapLineEntries.end() && !It->second.empty()) {719 // Check if the new line entry has the same debug info as the last one720 // to avoid duplicates. We don't compare labels since different721 // instructions can have the same line info.722 const auto &LastEntry = It->second.back();723 if (LastEntry.getFileNum() == NewLineEntry.getFileNum() &&724 LastEntry.getLine() == NewLineEntry.getLine() &&725 LastEntry.getColumn() == NewLineEntry.getColumn() &&726 LastEntry.getFlags() == NewLineEntry.getFlags() &&727 LastEntry.getIsa() == NewLineEntry.getIsa() &&728 LastEntry.getDiscriminator() == NewLineEntry.getDiscriminator())729 return;730 }731 732 BC.getDwarfLineTable(TargetUnitIndex)733 .getMCLineSections()734 .addLineEntry(NewLineEntry, Streamer.getCurrentSectionOnly());735 };736 737 if (!InstrLabel)738 InstrLabel = BC.Ctx->createTempSymbol();739 for (DebugLineTableRowRef RowReference : Cluster->getRows()) {740 const DWARFDebugLine::LineTable *CurrentLineTable =741 BC.DwCtx->getLineTableForUnit(742 BC.DwCtx->getCompileUnitForOffset(RowReference.DwCompileUnitIndex));743 const DWARFDebugLine::Row &CurrentRow =744 CurrentLineTable->Rows[RowReference.RowIndex - 1];745 unsigned Flags = (DWARF2_FLAG_IS_STMT * CurrentRow.IsStmt) |746 (DWARF2_FLAG_BASIC_BLOCK * CurrentRow.BasicBlock) |747 (DWARF2_FLAG_PROLOGUE_END * CurrentRow.PrologueEnd) |748 (DWARF2_FLAG_EPILOGUE_BEGIN * CurrentRow.EpilogueBegin);749 750 // Always emit is_stmt at the beginning of function fragment.751 if (FirstInstr)752 Flags |= DWARF2_FLAG_IS_STMT;753 const auto &FunctionDwarfUnits = BF.getDWARFUnits();754 auto It = FunctionDwarfUnits.find(RowReference.DwCompileUnitIndex);755 if (It != FunctionDwarfUnits.end()) {756 addToLineTable(RowReference, *It->second, Flags, *InstrLabel, CurrentRow);757 continue;758 }759 // This rows is from CU that did not contain the original function.760 // This might happen if BOLT moved/inlined that instruction from other CUs.761 // In this case, we need to insert it to all CUs that the function762 // originally beloned to.763 for (const auto &[_, Unit] : BF.getDWARFUnits()) {764 addToLineTable(RowReference, *Unit, Flags, *InstrLabel, CurrentRow);765 }766 }767 768 return NewLoc;769}770 771void BinaryEmitter::emitLineInfoEnd(const BinaryFunction &BF,772 MCSymbol *FunctionEndLabel,773 const DWARFUnit &Unit) {774 BC.Ctx->setCurrentDwarfLoc(0, 0, 0, DWARF2_FLAG_END_SEQUENCE, 0, 0);775 const MCDwarfLoc &DwarfLoc = BC.Ctx->getCurrentDwarfLoc();776 BC.Ctx->clearDwarfLocSeen();777 BC.getDwarfLineTable(Unit.getOffset())778 .getMCLineSections()779 .addLineEntry(MCDwarfLineEntry(FunctionEndLabel, DwarfLoc),780 Streamer.getCurrentSectionOnly());781}782 783void BinaryEmitter::emitJumpTables(const BinaryFunction &BF) {784 MCSection *ReadOnlySection = BC.MOFI->getReadOnlySection();785 MCSection *ReadOnlyColdSection = BC.MOFI->getContext().getELFSection(786 ".rodata.cold", ELF::SHT_PROGBITS, ELF::SHF_ALLOC);787 788 if (!BF.hasJumpTables())789 return;790 791 if (opts::PrintJumpTables)792 BC.outs() << "BOLT-INFO: jump tables for function " << BF << ":\n";793 794 for (auto &JTI : BF.jumpTables()) {795 JumpTable &JT = *JTI.second;796 // Only emit shared jump tables once, when processing the first parent797 if (JT.Parents.size() > 1 && JT.Parents[0] != &BF)798 continue;799 if (opts::PrintJumpTables)800 JT.print(BC.outs());801 if (opts::JumpTables == JTS_BASIC) {802 JT.updateOriginal();803 } else {804 MCSection *HotSection, *ColdSection;805 if (BF.isSimple()) {806 HotSection = ReadOnlySection;807 ColdSection = ReadOnlyColdSection;808 } else {809 HotSection = BF.hasProfile() ? ReadOnlySection : ReadOnlyColdSection;810 ColdSection = HotSection;811 }812 emitJumpTable(JT, HotSection, ColdSection);813 }814 }815}816 817void BinaryEmitter::emitJumpTable(const JumpTable &JT, MCSection *HotSection,818 MCSection *ColdSection) {819 // Pre-process entries for aggressive splitting.820 // Each label represents a separate switch table and gets its own count821 // determining its destination.822 std::map<MCSymbol *, uint64_t> LabelCounts;823 if (opts::JumpTables > JTS_SPLIT && !JT.Counts.empty()) {824 auto It = JT.Labels.find(0);825 assert(It != JT.Labels.end());826 MCSymbol *CurrentLabel = It->second;827 uint64_t CurrentLabelCount = 0;828 for (unsigned Index = 0; Index < JT.Entries.size(); ++Index) {829 auto LI = JT.Labels.find(Index * JT.EntrySize);830 if (LI != JT.Labels.end()) {831 LabelCounts[CurrentLabel] = CurrentLabelCount;832 CurrentLabel = LI->second;833 CurrentLabelCount = 0;834 }835 CurrentLabelCount += JT.Counts[Index].Count;836 }837 LabelCounts[CurrentLabel] = CurrentLabelCount;838 } else {839 Streamer.switchSection(JT.Count > 0 ? HotSection : ColdSection);840 Streamer.emitValueToAlignment(Align(JT.EntrySize));841 }842 MCSymbol *JTLabel = nullptr;843 uint64_t Offset = 0;844 for (MCSymbol *Entry : JT.Entries) {845 auto LI = JT.Labels.find(Offset);846 if (LI == JT.Labels.end())847 goto emitEntry;848 JTLabel = LI->second;849 LLVM_DEBUG({850 dbgs() << "BOLT-DEBUG: emitting jump table " << JTLabel->getName()851 << " (originally was at address 0x"852 << Twine::utohexstr(JT.getAddress() + Offset)853 << (Offset ? ") as part of larger jump table\n" : ")\n");854 });855 if (!LabelCounts.empty()) {856 const uint64_t JTCount = LabelCounts[JTLabel];857 LLVM_DEBUG(dbgs() << "BOLT-DEBUG: jump table count: " << JTCount << '\n');858 Streamer.switchSection(JTCount ? HotSection : ColdSection);859 Streamer.emitValueToAlignment(Align(JT.EntrySize));860 }861 // Emit all labels registered at the address of this jump table862 // to sync with our global symbol table. We may have two labels863 // registered at this address if one label was created via864 // getOrCreateGlobalSymbol() (e.g. LEA instructions referencing865 // this location) and another via getOrCreateJumpTable(). This866 // creates a race where the symbols created by these two867 // functions may or may not be the same, but they are both868 // registered in our symbol table at the same address. By869 // emitting them all here we make sure there is no ambiguity870 // that depends on the order that these symbols were created, so871 // whenever this address is referenced in the binary, it is872 // certain to point to the jump table identified at this873 // address.874 if (BinaryData *BD = BC.getBinaryDataByName(JTLabel->getName())) {875 for (MCSymbol *S : BD->getSymbols())876 Streamer.emitLabel(S);877 } else {878 Streamer.emitLabel(JTLabel);879 }880 emitEntry:881 if (JT.Type == JumpTable::JTT_NORMAL) {882 Streamer.emitSymbolValue(Entry, JT.OutputEntrySize);883 } else { // JTT_PIC884 const MCSymbolRefExpr *JTExpr =885 MCSymbolRefExpr::create(JTLabel, Streamer.getContext());886 const MCSymbolRefExpr *E =887 MCSymbolRefExpr::create(Entry, Streamer.getContext());888 const MCBinaryExpr *Value =889 MCBinaryExpr::createSub(E, JTExpr, Streamer.getContext());890 Streamer.emitValue(Value, JT.EntrySize);891 }892 Offset += JT.EntrySize;893 }894}895 896void BinaryEmitter::emitCFIInstruction(const MCCFIInstruction &Inst) const {897 switch (Inst.getOperation()) {898 default:899 llvm_unreachable("Unexpected instruction");900 case MCCFIInstruction::OpDefCfaOffset:901 Streamer.emitCFIDefCfaOffset(Inst.getOffset());902 break;903 case MCCFIInstruction::OpAdjustCfaOffset:904 Streamer.emitCFIAdjustCfaOffset(Inst.getOffset());905 break;906 case MCCFIInstruction::OpDefCfa:907 Streamer.emitCFIDefCfa(Inst.getRegister(), Inst.getOffset());908 break;909 case MCCFIInstruction::OpDefCfaRegister:910 Streamer.emitCFIDefCfaRegister(Inst.getRegister());911 break;912 case MCCFIInstruction::OpOffset:913 Streamer.emitCFIOffset(Inst.getRegister(), Inst.getOffset());914 break;915 case MCCFIInstruction::OpRegister:916 Streamer.emitCFIRegister(Inst.getRegister(), Inst.getRegister2());917 break;918 case MCCFIInstruction::OpWindowSave:919 Streamer.emitCFIWindowSave();920 break;921 case MCCFIInstruction::OpNegateRAState:922 Streamer.emitCFINegateRAState();923 break;924 case MCCFIInstruction::OpSameValue:925 Streamer.emitCFISameValue(Inst.getRegister());926 break;927 case MCCFIInstruction::OpGnuArgsSize:928 Streamer.emitCFIGnuArgsSize(Inst.getOffset());929 break;930 case MCCFIInstruction::OpEscape:931 Streamer.AddComment(Inst.getComment());932 Streamer.emitCFIEscape(Inst.getValues());933 break;934 case MCCFIInstruction::OpRestore:935 Streamer.emitCFIRestore(Inst.getRegister());936 break;937 case MCCFIInstruction::OpUndefined:938 Streamer.emitCFIUndefined(Inst.getRegister());939 break;940 }941}942 943// The code is based on EHStreamer::emitExceptionTable().944void BinaryEmitter::emitLSDA(BinaryFunction &BF, const FunctionFragment &FF) {945 const BinaryFunction::CallSitesRange Sites =946 BF.getCallSites(FF.getFragmentNum());947 if (Sites.empty())948 return;949 950 Streamer.switchSection(BC.MOFI->getLSDASection());951 952 const unsigned TTypeEncoding = BF.getLSDATypeEncoding();953 const unsigned TTypeEncodingSize = BC.getDWARFEncodingSize(TTypeEncoding);954 const uint16_t TTypeAlignment = 4;955 956 // Type tables have to be aligned at 4 bytes.957 Streamer.emitValueToAlignment(Align(TTypeAlignment));958 959 // Emit the LSDA label.960 MCSymbol *LSDASymbol = BF.getLSDASymbol(FF.getFragmentNum());961 assert(LSDASymbol && "no LSDA symbol set");962 Streamer.emitLabel(LSDASymbol);963 964 // Corresponding FDE start.965 const MCSymbol *StartSymbol = BF.getSymbol(FF.getFragmentNum());966 967 // Emit the LSDA header.968 969 // If LPStart is omitted, then the start of the FDE is used as a base for970 // landing pad displacements. Then, if a cold fragment starts with971 // a landing pad, this means that the first landing pad offset will be 0.972 // However, C++ runtime will treat 0 as if there is no landing pad, thus we973 // cannot emit LP offset as 0.974 //975 // As a solution, for fixed-address binaries we set LPStart to 0, and for976 // position-independent binaries we offset LP start by one byte.977 bool NeedsLPAdjustment = false;978 std::function<void(const MCSymbol *)> emitLandingPad;979 980 // Check if there's a symbol associated with a landing pad fragment.981 const MCSymbol *LPStartSymbol = BF.getLPStartSymbol(FF.getFragmentNum());982 if (!LPStartSymbol) {983 // Since landing pads are not in the same fragment, we fall back to emitting984 // absolute addresses for this FDE.985 if (opts::Verbosity >= 2) {986 BC.outs() << "BOLT-INFO: falling back to generating absolute-address "987 << "exception ranges for " << BF << '\n';988 }989 990 assert(BC.HasFixedLoadAddress &&991 "Cannot emit absolute-address landing pads for PIE/DSO");992 993 Streamer.emitIntValue(dwarf::DW_EH_PE_udata4, 1); // LPStart format994 Streamer.emitIntValue(0, 4); // LPStart995 emitLandingPad = [&](const MCSymbol *LPSymbol) {996 if (LPSymbol)997 Streamer.emitSymbolValue(LPSymbol, 4);998 else999 Streamer.emitIntValue(0, 4);1000 };1001 } else {1002 std::optional<FragmentNum> LPFN = BF.getLPFragment(FF.getFragmentNum());1003 const FunctionFragment &LPFragment = BF.getLayout().getFragment(*LPFN);1004 NeedsLPAdjustment =1005 (!LPFragment.empty() && LPFragment.front()->isLandingPad());1006 1007 // Emit LPStart encoding and optionally LPStart.1008 if (NeedsLPAdjustment || LPStartSymbol != StartSymbol) {1009 Streamer.emitIntValue(dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4, 1);1010 MCSymbol *DotSymbol = BC.Ctx->createTempSymbol("LPBase");1011 Streamer.emitLabel(DotSymbol);1012 1013 const MCExpr *LPStartExpr = MCBinaryExpr::createSub(1014 MCSymbolRefExpr::create(LPStartSymbol, *BC.Ctx),1015 MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx);1016 if (NeedsLPAdjustment)1017 LPStartExpr = MCBinaryExpr::createSub(1018 LPStartExpr, MCConstantExpr::create(1, *BC.Ctx), *BC.Ctx);1019 Streamer.emitValue(LPStartExpr, 4);1020 } else {1021 // DW_EH_PE_omit means FDE start (StartSymbol) will be used as LPStart.1022 Streamer.emitIntValue(dwarf::DW_EH_PE_omit, 1);1023 }1024 emitLandingPad = [&](const MCSymbol *LPSymbol) {1025 if (LPSymbol) {1026 const MCExpr *LPOffsetExpr = MCBinaryExpr::createSub(1027 MCSymbolRefExpr::create(LPSymbol, *BC.Ctx),1028 MCSymbolRefExpr::create(LPStartSymbol, *BC.Ctx), *BC.Ctx);1029 if (NeedsLPAdjustment)1030 LPOffsetExpr = MCBinaryExpr::createAdd(1031 LPOffsetExpr, MCConstantExpr::create(1, *BC.Ctx), *BC.Ctx);1032 Streamer.emitULEB128Value(LPOffsetExpr);1033 } else {1034 Streamer.emitULEB128IntValue(0);1035 }1036 };1037 }1038 1039 Streamer.emitIntValue(TTypeEncoding, 1); // TType format1040 1041 MCSymbol *TTBaseLabel = nullptr;1042 if (TTypeEncoding != dwarf::DW_EH_PE_omit) {1043 TTBaseLabel = BC.Ctx->createTempSymbol("TTBase");1044 MCSymbol *TTBaseRefLabel = BC.Ctx->createTempSymbol("TTBaseRef");1045 Streamer.emitAbsoluteSymbolDiffAsULEB128(TTBaseLabel, TTBaseRefLabel);1046 Streamer.emitLabel(TTBaseRefLabel);1047 }1048 1049 // Emit encoding of entries in the call site table. The format is used for the1050 // call site start, length, and corresponding landing pad.1051 if (!LPStartSymbol)1052 Streamer.emitIntValue(dwarf::DW_EH_PE_sdata4, 1);1053 else1054 Streamer.emitIntValue(dwarf::DW_EH_PE_uleb128, 1);1055 1056 MCSymbol *CSTStartLabel = BC.Ctx->createTempSymbol("CSTStart");1057 MCSymbol *CSTEndLabel = BC.Ctx->createTempSymbol("CSTEnd");1058 Streamer.emitAbsoluteSymbolDiffAsULEB128(CSTEndLabel, CSTStartLabel);1059 1060 Streamer.emitLabel(CSTStartLabel);1061 for (const auto &FragmentCallSite : Sites) {1062 const BinaryFunction::CallSite &CallSite = FragmentCallSite.second;1063 const MCSymbol *BeginLabel = CallSite.Start;1064 const MCSymbol *EndLabel = CallSite.End;1065 1066 assert(BeginLabel && "start EH label expected");1067 assert(EndLabel && "end EH label expected");1068 1069 // Start of the range is emitted relative to the start of current1070 // function split part.1071 if (!LPStartSymbol) {1072 Streamer.emitAbsoluteSymbolDiff(BeginLabel, StartSymbol, 4);1073 Streamer.emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 4);1074 } else {1075 Streamer.emitAbsoluteSymbolDiffAsULEB128(BeginLabel, StartSymbol);1076 Streamer.emitAbsoluteSymbolDiffAsULEB128(EndLabel, BeginLabel);1077 }1078 emitLandingPad(CallSite.LP);1079 Streamer.emitULEB128IntValue(CallSite.Action);1080 }1081 Streamer.emitLabel(CSTEndLabel);1082 1083 // Write out action, type, and type index tables at the end.1084 //1085 // For action and type index tables there's no need to change the original1086 // table format unless we are doing function splitting, in which case we can1087 // split and optimize the tables.1088 //1089 // For type table we (re-)encode the table using TTypeEncoding matching1090 // the current assembler mode.1091 for (uint8_t const &Byte : BF.getLSDAActionTable())1092 Streamer.emitIntValue(Byte, 1);1093 1094 const BinaryFunction::LSDATypeTableTy &TypeTable =1095 (TTypeEncoding & dwarf::DW_EH_PE_indirect) ? BF.getLSDATypeAddressTable()1096 : BF.getLSDATypeTable();1097 assert(TypeTable.size() == BF.getLSDATypeTable().size() &&1098 "indirect type table size mismatch");1099 1100 Streamer.emitValueToAlignment(Align(TTypeAlignment));1101 1102 for (int Index = TypeTable.size() - 1; Index >= 0; --Index) {1103 const uint64_t TypeAddress = TypeTable[Index];1104 switch (TTypeEncoding & 0x70) {1105 default:1106 llvm_unreachable("unsupported TTypeEncoding");1107 case dwarf::DW_EH_PE_absptr:1108 Streamer.emitIntValue(TypeAddress, TTypeEncodingSize);1109 break;1110 case dwarf::DW_EH_PE_pcrel: {1111 if (TypeAddress) {1112 const MCSymbol *TypeSymbol =1113 BC.getOrCreateGlobalSymbol(TypeAddress, "TI", 0, TTypeAlignment);1114 MCSymbol *DotSymbol = BC.Ctx->createNamedTempSymbol();1115 Streamer.emitLabel(DotSymbol);1116 const MCBinaryExpr *SubDotExpr = MCBinaryExpr::createSub(1117 MCSymbolRefExpr::create(TypeSymbol, *BC.Ctx),1118 MCSymbolRefExpr::create(DotSymbol, *BC.Ctx), *BC.Ctx);1119 Streamer.emitValue(SubDotExpr, TTypeEncodingSize);1120 } else {1121 Streamer.emitIntValue(0, TTypeEncodingSize);1122 }1123 break;1124 }1125 }1126 }1127 1128 if (TTypeEncoding != dwarf::DW_EH_PE_omit)1129 Streamer.emitLabel(TTBaseLabel);1130 1131 for (uint8_t const &Byte : BF.getLSDATypeIndexTable())1132 Streamer.emitIntValue(Byte, 1);1133}1134 1135void BinaryEmitter::emitDebugLineInfoForOriginalFunctions() {1136 // If a function is in a CU containing at least one processed function, we1137 // have to rewrite the whole line table for that CU. For unprocessed functions1138 // we use data from the input line table.1139 for (auto &It : BC.getBinaryFunctions()) {1140 const BinaryFunction &Function = It.second;1141 1142 // If the function was emitted, its line info was emitted with it.1143 if (Function.isEmitted())1144 continue;1145 1146 // Loop through all CUs in the function1147 for (const auto &[_, Unit] : Function.getDWARFUnits()) {1148 const DWARFDebugLine::LineTable *LineTable =1149 Function.getDWARFLineTableForUnit(Unit);1150 if (!LineTable)1151 continue; // nothing to update for this unit1152 1153 const uint64_t Address = Function.getAddress();1154 std::vector<uint32_t> Results;1155 if (!LineTable->lookupAddressRange(1156 {Address, object::SectionedAddress::UndefSection},1157 Function.getSize(), Results))1158 continue;1159 1160 if (Results.empty())1161 continue;1162 1163 // The first row returned could be the last row matching the start1164 // address. Find the first row with the same address that is not the end1165 // of the sequence.1166 uint64_t FirstRow = Results.front();1167 while (FirstRow > 0) {1168 const DWARFDebugLine::Row &PrevRow = LineTable->Rows[FirstRow - 1];1169 if (PrevRow.Address.Address != Address || PrevRow.EndSequence)1170 break;1171 --FirstRow;1172 }1173 1174 const uint64_t EndOfSequenceAddress =1175 Function.getAddress() + Function.getMaxSize();1176 BC.getDwarfLineTable(Unit->getOffset())1177 .addLineTableSequence(LineTable, FirstRow, Results.back(),1178 EndOfSequenceAddress);1179 }1180 }1181 1182 // For units that are completely unprocessed, use original debug line contents1183 // eliminating the need to regenerate line info program.1184 emitDebugLineInfoForUnprocessedCUs();1185}1186 1187void BinaryEmitter::emitDebugLineInfoForUnprocessedCUs() {1188 // Sorted list of section offsets provides boundaries for section fragments,1189 // where each fragment is the unit's contribution to debug line section.1190 std::vector<uint64_t> StmtListOffsets;1191 StmtListOffsets.reserve(BC.DwCtx->getNumCompileUnits());1192 for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {1193 DWARFDie CUDie = CU->getUnitDIE();1194 auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));1195 if (!StmtList)1196 continue;1197 1198 StmtListOffsets.push_back(*StmtList);1199 }1200 llvm::sort(StmtListOffsets);1201 1202 // For each CU that was not processed, emit its line info as a binary blob.1203 for (const std::unique_ptr<DWARFUnit> &CU : BC.DwCtx->compile_units()) {1204 if (BC.ProcessedCUs.count(CU.get()))1205 continue;1206 1207 DWARFDie CUDie = CU->getUnitDIE();1208 auto StmtList = dwarf::toSectionOffset(CUDie.find(dwarf::DW_AT_stmt_list));1209 if (!StmtList)1210 continue;1211 1212 StringRef DebugLineContents = CU->getLineSection().Data;1213 1214 const uint64_t Begin = *StmtList;1215 1216 // Statement list ends where the next unit contribution begins, or at the1217 // end of the section.1218 auto It = llvm::upper_bound(StmtListOffsets, Begin);1219 const uint64_t End =1220 It == StmtListOffsets.end() ? DebugLineContents.size() : *It;1221 1222 BC.getDwarfLineTable(CU->getOffset())1223 .addRawContents(DebugLineContents.slice(Begin, End));1224 }1225}1226 1227void BinaryEmitter::emitDataSections(StringRef OrgSecPrefix) {1228 for (BinarySection &Section : BC.sections()) {1229 if (!Section.hasRelocations())1230 continue;1231 1232 StringRef Prefix = Section.hasSectionRef() ? OrgSecPrefix : "";1233 Section.emitAsData(Streamer, Prefix + Section.getName());1234 Section.clearRelocations();1235 }1236}1237 1238namespace llvm {1239namespace bolt {1240 1241void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,1242 StringRef OrgSecPrefix) {1243 BinaryEmitter(Streamer, BC).emitAll(OrgSecPrefix);1244}1245 1246void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,1247 FunctionFragment &FF, bool EmitCodeOnly) {1248 BinaryEmitter(Streamer, BF.getBinaryContext())1249 .emitFunctionBody(BF, FF, EmitCodeOnly);1250}1251 1252} // namespace bolt1253} // namespace llvm1254