1084 lines · cpp
1//===-- ProfiledBinary.cpp - Binary decoder ---------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "ProfiledBinary.h"10#include "ErrorHandling.h"11#include "MissingFrameInferrer.h"12#include "Options.h"13#include "ProfileGenerator.h"14#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"15#include "llvm/Demangle/Demangle.h"16#include "llvm/IR/DebugInfoMetadata.h"17#include "llvm/MC/TargetRegistry.h"18#include "llvm/Object/COFF.h"19#include "llvm/Support/CommandLine.h"20#include "llvm/Support/Debug.h"21#include "llvm/Support/Format.h"22#include "llvm/Support/TargetSelect.h"23#include "llvm/TargetParser/Triple.h"24#include <optional>25 26#define DEBUG_TYPE "load-binary"27 28namespace llvm {29 30using namespace object;31 32cl::opt<bool> ShowDisassemblyOnly("show-disassembly-only",33 cl::desc("Print disassembled code."),34 cl::cat(ProfGenCategory));35 36cl::opt<bool> ShowSourceLocations("show-source-locations",37 cl::desc("Print source locations."),38 cl::cat(ProfGenCategory));39 40static cl::opt<bool>41 ShowCanonicalFnName("show-canonical-fname",42 cl::desc("Print canonical function name."),43 cl::cat(ProfGenCategory));44 45static cl::opt<bool> ShowPseudoProbe(46 "show-pseudo-probe",47 cl::desc("Print pseudo probe section and disassembled info."),48 cl::cat(ProfGenCategory));49 50static cl::opt<bool> UseDwarfCorrelation(51 "use-dwarf-correlation",52 cl::desc("Use dwarf for profile correlation even when binary contains "53 "pseudo probe."),54 cl::cat(ProfGenCategory));55 56static cl::opt<std::string>57 DWPPath("dwp", cl::init(""),58 cl::desc("Path of .dwp file. When not specified, it will be "59 "<binary>.dwp in the same directory as the main binary."),60 cl::cat(ProfGenCategory));61 62static cl::list<std::string> DisassembleFunctions(63 "disassemble-functions", cl::CommaSeparated,64 cl::desc("List of functions to print disassembly for. Accept demangled "65 "names only. Only work with show-disassembly-only"),66 cl::cat(ProfGenCategory));67 68static cl::opt<bool>69 KernelBinary("kernel",70 cl::desc("Generate the profile for Linux kernel binary."),71 cl::cat(ProfGenCategory));72 73namespace sampleprof {74 75static const Target *getTarget(const ObjectFile *Obj) {76 Triple TheTriple = Obj->makeTriple();77 std::string Error;78 std::string ArchName;79 const Target *TheTarget =80 TargetRegistry::lookupTarget(ArchName, TheTriple, Error);81 if (!TheTarget)82 exitWithError(Error, Obj->getFileName());83 return TheTarget;84}85 86void BinarySizeContextTracker::addInstructionForContext(87 const SampleContextFrameVector &Context, uint32_t InstrSize) {88 ContextTrieNode *CurNode = &RootContext;89 bool IsLeaf = true;90 for (const auto &Callsite : reverse(Context)) {91 FunctionId CallerName = Callsite.Func;92 LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location;93 CurNode = CurNode->getOrCreateChildContext(CallsiteLoc, CallerName);94 IsLeaf = false;95 }96 97 CurNode->addFunctionSize(InstrSize);98}99 100uint32_t101BinarySizeContextTracker::getFuncSizeForContext(const ContextTrieNode *Node) {102 ContextTrieNode *CurrNode = &RootContext;103 ContextTrieNode *PrevNode = nullptr;104 105 std::optional<uint32_t> Size;106 107 // Start from top-level context-less function, traverse down the reverse108 // context trie to find the best/longest match for given context, then109 // retrieve the size.110 LineLocation CallSiteLoc(0, 0);111 while (CurrNode && Node->getParentContext() != nullptr) {112 PrevNode = CurrNode;113 CurrNode = CurrNode->getChildContext(CallSiteLoc, Node->getFuncName());114 if (CurrNode && CurrNode->getFunctionSize())115 Size = *CurrNode->getFunctionSize();116 CallSiteLoc = Node->getCallSiteLoc();117 Node = Node->getParentContext();118 }119 120 // If we traversed all nodes along the path of the context and haven't121 // found a size yet, pivot to look for size from sibling nodes, i.e size122 // of inlinee under different context.123 if (!Size) {124 if (!CurrNode)125 CurrNode = PrevNode;126 while (!Size && CurrNode && !CurrNode->getAllChildContext().empty()) {127 CurrNode = &CurrNode->getAllChildContext().begin()->second;128 if (CurrNode->getFunctionSize())129 Size = *CurrNode->getFunctionSize();130 }131 }132 133 assert(Size && "We should at least find one context size.");134 return *Size;135}136 137void BinarySizeContextTracker::trackInlineesOptimizedAway(138 MCPseudoProbeDecoder &ProbeDecoder) {139 ProbeFrameStack ProbeContext;140 for (const auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren())141 trackInlineesOptimizedAway(ProbeDecoder, Child, ProbeContext);142}143 144void BinarySizeContextTracker::trackInlineesOptimizedAway(145 MCPseudoProbeDecoder &ProbeDecoder,146 const MCDecodedPseudoProbeInlineTree &ProbeNode,147 ProbeFrameStack &ProbeContext) {148 StringRef FuncName =149 ProbeDecoder.getFuncDescForGUID(ProbeNode.Guid)->FuncName;150 ProbeContext.emplace_back(FuncName, 0);151 152 // This ProbeContext has a probe, so it has code before inlining and153 // optimization. Make sure we mark its size as known.154 if (!ProbeNode.getProbes().empty()) {155 ContextTrieNode *SizeContext = &RootContext;156 for (auto &ProbeFrame : reverse(ProbeContext)) {157 StringRef CallerName = ProbeFrame.first;158 LineLocation CallsiteLoc(ProbeFrame.second, 0);159 SizeContext =160 SizeContext->getOrCreateChildContext(CallsiteLoc,161 FunctionId(CallerName));162 }163 // Add 0 size to make known.164 SizeContext->addFunctionSize(0);165 }166 167 // DFS down the probe inline tree168 for (const auto &ChildNode : ProbeNode.getChildren()) {169 InlineSite Location = ChildNode.getInlineSite();170 ProbeContext.back().second = std::get<1>(Location);171 trackInlineesOptimizedAway(ProbeDecoder, ChildNode, ProbeContext);172 }173 174 ProbeContext.pop_back();175}176 177ProfiledBinary::ProfiledBinary(const StringRef ExeBinPath,178 const StringRef DebugBinPath)179 : Path(ExeBinPath), DebugBinaryPath(DebugBinPath),180 SymbolizerOpts(getSymbolizerOpts()), ProEpilogTracker(this),181 Symbolizer(std::make_unique<symbolize::LLVMSymbolizer>(SymbolizerOpts)),182 TrackFuncContextSize(EnableCSPreInliner && UseContextCostForPreInliner) {183 // Point to executable binary if debug info binary is not specified.184 SymbolizerPath = DebugBinPath.empty() ? ExeBinPath : DebugBinPath;185 if (InferMissingFrames)186 MissingContextInferrer = std::make_unique<MissingFrameInferrer>(this);187 load();188}189 190ProfiledBinary::~ProfiledBinary() = default;191 192void ProfiledBinary::warnNoFuncEntry() {193 uint64_t NoFuncEntryNum = 0;194 for (auto &F : BinaryFunctions) {195 if (F.second.Ranges.empty())196 continue;197 bool hasFuncEntry = false;198 for (auto &R : F.second.Ranges) {199 if (FuncRange *FR = findFuncRangeForStartAddr(R.first)) {200 if (FR->IsFuncEntry) {201 hasFuncEntry = true;202 break;203 }204 }205 }206 207 if (!hasFuncEntry) {208 NoFuncEntryNum++;209 if (ShowDetailedWarning)210 WithColor::warning()211 << "Failed to determine function entry for " << F.first212 << " due to inconsistent name from symbol table and dwarf info.\n";213 }214 }215 emitWarningSummary(NoFuncEntryNum, BinaryFunctions.size(),216 "of functions failed to determine function entry due to "217 "inconsistent name from symbol table and dwarf info.");218}219 220void ProfiledBinary::load() {221 // Attempt to open the binary.222 OwningBinary<Binary> OBinary = unwrapOrError(createBinary(Path), Path);223 Binary &ExeBinary = *OBinary.getBinary();224 225 IsCOFF = isa<COFFObjectFile>(&ExeBinary);226 if (!isa<ELFObjectFileBase>(&ExeBinary) && !IsCOFF)227 exitWithError("not a valid ELF/COFF image", Path);228 229 auto *Obj = cast<ObjectFile>(&ExeBinary);230 TheTriple = Obj->makeTriple();231 232 LLVM_DEBUG(dbgs() << "Loading " << Path << "\n");233 234 // Mark the binary as a kernel image;235 IsKernel = KernelBinary;236 237 // Find the preferred load address for text sections.238 setPreferredTextSegmentAddresses(Obj);239 240 // Load debug info of subprograms from DWARF section.241 // If path of debug info binary is specified, use the debug info from it,242 // otherwise use the debug info from the executable binary.243 if (!DebugBinaryPath.empty()) {244 OwningBinary<Binary> DebugPath =245 unwrapOrError(createBinary(DebugBinaryPath), DebugBinaryPath);246 loadSymbolsFromDWARF(*cast<ObjectFile>(DebugPath.getBinary()));247 } else {248 loadSymbolsFromDWARF(*cast<ObjectFile>(&ExeBinary));249 }250 251 DisassembleFunctionSet.insert_range(DisassembleFunctions);252 253 checkPseudoProbe(Obj);254 if (UsePseudoProbes)255 populateSymbolAddressList(Obj);256 257 if (ShowDisassemblyOnly)258 decodePseudoProbe(Obj);259 260 // Disassemble the text sections.261 disassemble(Obj);262 263 // Use function start and return address to infer prolog and epilog264 ProEpilogTracker.inferPrologAddresses(StartAddrToFuncRangeMap);265 ProEpilogTracker.inferEpilogAddresses(RetAddressSet);266 267 warnNoFuncEntry();268 269 // TODO: decode other sections.270}271 272bool ProfiledBinary::inlineContextEqual(uint64_t Address1, uint64_t Address2) {273 const SampleContextFrameVector &Context1 =274 getCachedFrameLocationStack(Address1);275 const SampleContextFrameVector &Context2 =276 getCachedFrameLocationStack(Address2);277 if (Context1.size() != Context2.size())278 return false;279 if (Context1.empty())280 return false;281 // The leaf frame contains location within the leaf, and it282 // needs to be remove that as it's not part of the calling context283 return std::equal(Context1.begin(), Context1.begin() + Context1.size() - 1,284 Context2.begin(), Context2.begin() + Context2.size() - 1);285}286 287SampleContextFrameVector288ProfiledBinary::getExpandedContext(const SmallVectorImpl<uint64_t> &Stack,289 bool &WasLeafInlined) {290 SampleContextFrameVector ContextVec;291 if (Stack.empty())292 return ContextVec;293 // Process from frame root to leaf294 for (auto Address : Stack) {295 const SampleContextFrameVector &ExpandedContext =296 getCachedFrameLocationStack(Address);297 // An instruction without a valid debug line will be ignored by sample298 // processing299 if (ExpandedContext.empty())300 return SampleContextFrameVector();301 // Set WasLeafInlined to the size of inlined frame count for the last302 // address which is leaf303 WasLeafInlined = (ExpandedContext.size() > 1);304 ContextVec.append(ExpandedContext);305 }306 307 // Replace with decoded base discriminator308 for (auto &Frame : ContextVec) {309 Frame.Location.Discriminator = ProfileGeneratorBase::getBaseDiscriminator(310 Frame.Location.Discriminator, UseFSDiscriminator);311 }312 313 assert(ContextVec.size() && "Context length should be at least 1");314 315 // Compress the context string except for the leaf frame316 auto LeafFrame = ContextVec.back();317 LeafFrame.Location = LineLocation(0, 0);318 ContextVec.pop_back();319 CSProfileGenerator::compressRecursionContext(ContextVec);320 CSProfileGenerator::trimContext(ContextVec);321 ContextVec.push_back(LeafFrame);322 return ContextVec;323}324 325template <class ELFT>326void ProfiledBinary::setPreferredTextSegmentAddresses(const ELFFile<ELFT> &Obj,327 StringRef FileName) {328 const auto &PhdrRange = unwrapOrError(Obj.program_headers(), FileName);329 // FIXME: This should be the page size of the system running profiling.330 // However such info isn't available at post-processing time, assuming331 // 4K page now. Note that we don't use EXEC_PAGESIZE from <linux/param.h>332 // because we may build the tools on non-linux.333 uint64_t PageSize = 0x1000;334 for (const typename ELFT::Phdr &Phdr : PhdrRange) {335 if (Phdr.p_type == ELF::PT_LOAD) {336 if (!FirstLoadableAddress)337 FirstLoadableAddress = Phdr.p_vaddr & ~(PageSize - 1U);338 if (Phdr.p_flags & ELF::PF_X) {339 // Segments will always be loaded at a page boundary.340 PreferredTextSegmentAddresses.push_back(Phdr.p_vaddr &341 ~(PageSize - 1U));342 TextSegmentOffsets.push_back(Phdr.p_offset & ~(PageSize - 1U));343 } else {344 PhdrInfo Info;345 Info.FileOffset = Phdr.p_offset;346 Info.FileSz = Phdr.p_filesz;347 Info.VirtualAddr = Phdr.p_vaddr;348 NonTextPhdrInfo.push_back(Info);349 }350 }351 }352 353 if (PreferredTextSegmentAddresses.empty())354 exitWithError("no executable segment found", FileName);355}356 357uint64_t ProfiledBinary::CanonicalizeNonTextAddress(uint64_t Address) {358 uint64_t FileOffset = 0;359 auto MMapIter = NonTextMMapEvents.lower_bound(Address);360 if (MMapIter == NonTextMMapEvents.end())361 return Address; // No non-text mmap event found, return the address as is.362 363 const auto &MMapEvent = MMapIter->second;364 365 // If the address is within the non-text mmap event, calculate its file366 // offset in the binary.367 if (MMapEvent.Address <= Address &&368 Address < MMapEvent.Address + MMapEvent.Size)369 FileOffset = Address - MMapEvent.Address + MMapEvent.Offset;370 371 // If the address is not within the non-text mmap event, return the address372 // as is.373 if (FileOffset == 0)374 return Address;375 376 for (const auto &PhdrInfo : NonTextPhdrInfo) {377 // Find the program section that contains the file offset and map the378 // file offset to the virtual address.379 if (PhdrInfo.FileOffset <= FileOffset &&380 FileOffset < PhdrInfo.FileOffset + PhdrInfo.FileSz)381 return PhdrInfo.VirtualAddr + (FileOffset - PhdrInfo.FileOffset);382 }383 384 return Address;385}386 387void ProfiledBinary::setPreferredTextSegmentAddresses(const COFFObjectFile *Obj,388 StringRef FileName) {389 uint64_t ImageBase = Obj->getImageBase();390 if (!ImageBase)391 exitWithError("Not a COFF image", FileName);392 393 PreferredTextSegmentAddresses.push_back(ImageBase);394 FirstLoadableAddress = ImageBase;395 396 for (SectionRef Section : Obj->sections()) {397 const coff_section *Sec = Obj->getCOFFSection(Section);398 if (Sec->Characteristics & COFF::IMAGE_SCN_CNT_CODE)399 TextSegmentOffsets.push_back(Sec->VirtualAddress);400 }401}402 403void ProfiledBinary::setPreferredTextSegmentAddresses(const ObjectFile *Obj) {404 if (const auto *ELFObj = dyn_cast<ELF32LEObjectFile>(Obj))405 setPreferredTextSegmentAddresses(ELFObj->getELFFile(), Obj->getFileName());406 else if (const auto *ELFObj = dyn_cast<ELF32BEObjectFile>(Obj))407 setPreferredTextSegmentAddresses(ELFObj->getELFFile(), Obj->getFileName());408 else if (const auto *ELFObj = dyn_cast<ELF64LEObjectFile>(Obj))409 setPreferredTextSegmentAddresses(ELFObj->getELFFile(), Obj->getFileName());410 else if (const auto *ELFObj = dyn_cast<ELF64BEObjectFile>(Obj))411 setPreferredTextSegmentAddresses(ELFObj->getELFFile(), Obj->getFileName());412 else if (const auto *COFFObj = dyn_cast<COFFObjectFile>(Obj))413 setPreferredTextSegmentAddresses(COFFObj, Obj->getFileName());414 else415 llvm_unreachable("invalid object format");416}417 418void ProfiledBinary::checkPseudoProbe(const ObjectFile *Obj) {419 if (UseDwarfCorrelation)420 return;421 422 bool HasProbeDescSection = false;423 bool HasPseudoProbeSection = false;424 425 StringRef FileName = Obj->getFileName();426 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();427 SI != SE; ++SI) {428 const SectionRef &Section = *SI;429 StringRef SectionName = unwrapOrError(Section.getName(), FileName);430 if (SectionName == ".pseudo_probe_desc") {431 HasProbeDescSection = true;432 } else if (SectionName == ".pseudo_probe") {433 HasPseudoProbeSection = true;434 }435 }436 437 // set UsePseudoProbes flag, used for PerfReader438 UsePseudoProbes = HasProbeDescSection && HasPseudoProbeSection;439}440 441void ProfiledBinary::decodePseudoProbe(const ObjectFile *Obj) {442 if (!UsePseudoProbes)443 return;444 445 MCPseudoProbeDecoder::Uint64Set GuidFilter;446 MCPseudoProbeDecoder::Uint64Map FuncStartAddresses;447 if (ShowDisassemblyOnly) {448 if (DisassembleFunctionSet.empty()) {449 FuncStartAddresses = SymbolStartAddrs;450 } else {451 for (auto &F : DisassembleFunctionSet) {452 auto GUID = Function::getGUIDAssumingExternalLinkage(F.first());453 if (auto StartAddr = SymbolStartAddrs.lookup(GUID)) {454 FuncStartAddresses[GUID] = StartAddr;455 FuncRange &Range = StartAddrToFuncRangeMap[StartAddr];456 GuidFilter.insert(457 Function::getGUIDAssumingExternalLinkage(Range.getFuncName()));458 }459 }460 }461 } else {462 for (auto *F : ProfiledFunctions) {463 GuidFilter.insert(Function::getGUIDAssumingExternalLinkage(F->FuncName));464 for (auto &Range : F->Ranges) {465 auto GUIDs = StartAddrToSymMap.equal_range(Range.first);466 for (const auto &[StartAddr, Func] : make_range(GUIDs))467 FuncStartAddresses[Func] = StartAddr;468 }469 }470 }471 472 StringRef FileName = Obj->getFileName();473 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();474 SI != SE; ++SI) {475 const SectionRef &Section = *SI;476 StringRef SectionName = unwrapOrError(Section.getName(), FileName);477 478 if (SectionName == ".pseudo_probe_desc") {479 StringRef Contents = unwrapOrError(Section.getContents(), FileName);480 if (!ProbeDecoder.buildGUID2FuncDescMap(481 reinterpret_cast<const uint8_t *>(Contents.data()),482 Contents.size()))483 exitWithError(484 "Pseudo Probe decoder fail in .pseudo_probe_desc section");485 } else if (SectionName == ".pseudo_probe") {486 StringRef Contents = unwrapOrError(Section.getContents(), FileName);487 if (!ProbeDecoder.buildAddress2ProbeMap(488 reinterpret_cast<const uint8_t *>(Contents.data()),489 Contents.size(), GuidFilter, FuncStartAddresses))490 exitWithError("Pseudo Probe decoder fail in .pseudo_probe section");491 }492 }493 494 // Build TopLevelProbeFrameMap to track size for optimized inlinees when probe495 // is available496 if (TrackFuncContextSize) {497 for (auto &Child : ProbeDecoder.getDummyInlineRoot().getChildren()) {498 auto *Frame = &Child;499 StringRef FuncName =500 ProbeDecoder.getFuncDescForGUID(Frame->Guid)->FuncName;501 TopLevelProbeFrameMap[FuncName] = Frame;502 }503 }504 505 if (ShowPseudoProbe)506 ProbeDecoder.printGUID2FuncDescMap(outs());507}508 509void ProfiledBinary::decodePseudoProbe() {510 OwningBinary<Binary> OBinary = unwrapOrError(createBinary(Path), Path);511 Binary &ExeBinary = *OBinary.getBinary();512 auto *Obj = cast<ObjectFile>(&ExeBinary);513 decodePseudoProbe(Obj);514}515 516void ProfiledBinary::setIsFuncEntry(FuncRange *FuncRange,517 StringRef RangeSymName) {518 // Skip external function symbol.519 if (!FuncRange)520 return;521 522 // Set IsFuncEntry to ture if there is only one range in the function or the523 // RangeSymName from ELF is equal to its DWARF-based function name.524 if (FuncRange->Func->Ranges.size() == 1 ||525 (!FuncRange->IsFuncEntry && FuncRange->getFuncName() == RangeSymName))526 FuncRange->IsFuncEntry = true;527}528 529bool ProfiledBinary::dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,530 SectionSymbolsTy &Symbols,531 const SectionRef &Section) {532 std::size_t SE = Symbols.size();533 uint64_t SectionAddress = Section.getAddress();534 uint64_t SectSize = Section.getSize();535 uint64_t StartAddress = Symbols[SI].Addr;536 uint64_t NextStartAddress =537 (SI + 1 < SE) ? Symbols[SI + 1].Addr : SectionAddress + SectSize;538 FuncRange *FRange = findFuncRange(StartAddress);539 setIsFuncEntry(FRange, FunctionSamples::getCanonicalFnName(Symbols[SI].Name));540 StringRef SymbolName =541 ShowCanonicalFnName542 ? FunctionSamples::getCanonicalFnName(Symbols[SI].Name)543 : Symbols[SI].Name;544 bool ShowDisassembly =545 ShowDisassemblyOnly && (DisassembleFunctionSet.empty() ||546 DisassembleFunctionSet.count(SymbolName));547 if (ShowDisassembly)548 outs() << '<' << SymbolName << ">:\n";549 550 uint64_t Address = StartAddress;551 // Size of a consecutive invalid instruction range starting from Address -1552 // backwards.553 uint64_t InvalidInstLength = 0;554 while (Address < NextStartAddress) {555 MCInst Inst;556 uint64_t Size;557 // Disassemble an instruction.558 bool Disassembled = DisAsm->getInstruction(559 Inst, Size, Bytes.slice(Address - SectionAddress), Address, nulls());560 if (Size == 0)561 Size = 1;562 563 if (ShowDisassembly) {564 if (ShowPseudoProbe) {565 ProbeDecoder.printProbeForAddress(outs(), Address);566 }567 outs() << format("%8" PRIx64 ":", Address);568 size_t Start = outs().tell();569 if (Disassembled)570 IPrinter->printInst(&Inst, Address + Size, "", *STI, outs());571 else572 outs() << "\t<unknown>";573 if (ShowSourceLocations) {574 unsigned Cur = outs().tell() - Start;575 if (Cur < 40)576 outs().indent(40 - Cur);577 InstructionPointer IP(this, Address);578 outs() << getReversedLocWithContext(579 symbolize(IP, ShowCanonicalFnName, ShowPseudoProbe));580 }581 outs() << "\n";582 }583 584 if (Disassembled) {585 const MCInstrDesc &MCDesc = MII->get(Inst.getOpcode());586 587 // Record instruction size.588 AddressToInstSizeMap[Address] = Size;589 590 // Populate address maps.591 CodeAddressVec.push_back(Address);592 if (MCDesc.isCall()) {593 CallAddressSet.insert(Address);594 UncondBranchAddrSet.insert(Address);595 } else if (MCDesc.isReturn()) {596 RetAddressSet.insert(Address);597 UncondBranchAddrSet.insert(Address);598 } else if (MCDesc.isBranch()) {599 if (MCDesc.isUnconditionalBranch())600 UncondBranchAddrSet.insert(Address);601 BranchAddressSet.insert(Address);602 }603 604 // Record potential call targets for tail frame inference later-on.605 if (InferMissingFrames && FRange) {606 uint64_t Target = 0;607 MIA->evaluateBranch(Inst, Address, Size, Target);608 if (MCDesc.isCall()) {609 // Indirect call targets are unknown at this point. Recording the610 // unknown target (zero) for further LBR-based refinement.611 MissingContextInferrer->CallEdges[Address].insert(Target);612 } else if (MCDesc.isUnconditionalBranch()) {613 assert(Target &&614 "target should be known for unconditional direct branch");615 // Any inter-function unconditional jump is considered tail call at616 // this point. This is not 100% accurate and could further be617 // optimized based on some source annotation.618 FuncRange *ToFRange = findFuncRange(Target);619 if (ToFRange && ToFRange->Func != FRange->Func)620 MissingContextInferrer->TailCallEdges[Address].insert(Target);621 LLVM_DEBUG({622 dbgs() << "Direct Tail call: " << format("%8" PRIx64 ":", Address);623 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());624 dbgs() << "\n";625 });626 } else if (MCDesc.isIndirectBranch() && MCDesc.isBarrier()) {627 // This is an indirect branch but not necessarily an indirect tail628 // call. The isBarrier check is to filter out conditional branch.629 // Similar with indirect call targets, recording the unknown target630 // (zero) for further LBR-based refinement.631 MissingContextInferrer->TailCallEdges[Address].insert(Target);632 LLVM_DEBUG({633 dbgs() << "Indirect Tail call: "634 << format("%8" PRIx64 ":", Address);635 IPrinter->printInst(&Inst, Address + Size, "", *STI.get(), dbgs());636 dbgs() << "\n";637 });638 }639 }640 641 if (InvalidInstLength) {642 AddrsWithInvalidInstruction.insert(643 {Address - InvalidInstLength, Address - 1});644 InvalidInstLength = 0;645 }646 } else {647 InvalidInstLength += Size;648 }649 650 Address += Size;651 }652 653 if (InvalidInstLength)654 AddrsWithInvalidInstruction.insert(655 {Address - InvalidInstLength, Address - 1});656 657 if (ShowDisassembly)658 outs() << "\n";659 660 return true;661}662 663void ProfiledBinary::setUpDisassembler(const ObjectFile *Obj) {664 const Target *TheTarget = getTarget(Obj);665 StringRef FileName = Obj->getFileName();666 667 MRI.reset(TheTarget->createMCRegInfo(TheTriple));668 if (!MRI)669 exitWithError("no register info for target " + TheTriple.str(), FileName);670 671 MCTargetOptions MCOptions;672 AsmInfo.reset(TheTarget->createMCAsmInfo(*MRI, TheTriple, MCOptions));673 if (!AsmInfo)674 exitWithError("no assembly info for target " + TheTriple.str(), FileName);675 676 Expected<SubtargetFeatures> Features = Obj->getFeatures();677 if (!Features)678 exitWithError(Features.takeError(), FileName);679 STI.reset(680 TheTarget->createMCSubtargetInfo(TheTriple, "", Features->getString()));681 if (!STI)682 exitWithError("no subtarget info for target " + TheTriple.str(), FileName);683 684 MII.reset(TheTarget->createMCInstrInfo());685 if (!MII)686 exitWithError("no instruction info for target " + TheTriple.str(),687 FileName);688 689 MCContext Ctx(TheTriple, AsmInfo.get(), MRI.get(), STI.get());690 std::unique_ptr<MCObjectFileInfo> MOFI(691 TheTarget->createMCObjectFileInfo(Ctx, /*PIC=*/false));692 Ctx.setObjectFileInfo(MOFI.get());693 DisAsm.reset(TheTarget->createMCDisassembler(*STI, Ctx));694 if (!DisAsm)695 exitWithError("no disassembler for target " + TheTriple.str(), FileName);696 697 MIA.reset(TheTarget->createMCInstrAnalysis(MII.get()));698 699 int AsmPrinterVariant = AsmInfo->getAssemblerDialect();700 IPrinter.reset(TheTarget->createMCInstPrinter(TheTriple, AsmPrinterVariant,701 *AsmInfo, *MII, *MRI));702 IPrinter->setPrintBranchImmAsAddress(true);703}704 705void ProfiledBinary::disassemble(const ObjectFile *Obj) {706 // Set up disassembler and related components.707 setUpDisassembler(Obj);708 709 // Create a mapping from virtual address to symbol name. The symbols in text710 // sections are the candidates to dissassemble.711 std::map<SectionRef, SectionSymbolsTy> AllSymbols;712 StringRef FileName = Obj->getFileName();713 for (const SymbolRef &Symbol : Obj->symbols()) {714 const uint64_t Addr = unwrapOrError(Symbol.getAddress(), FileName);715 const StringRef Name = unwrapOrError(Symbol.getName(), FileName);716 section_iterator SecI = unwrapOrError(Symbol.getSection(), FileName);717 if (SecI != Obj->section_end())718 AllSymbols[*SecI].push_back(SymbolInfoTy(Addr, Name, ELF::STT_NOTYPE));719 }720 721 // Sort all the symbols. Use a stable sort to stabilize the output.722 for (std::pair<const SectionRef, SectionSymbolsTy> &SecSyms : AllSymbols)723 stable_sort(SecSyms.second);724 725 assert((DisassembleFunctionSet.empty() || ShowDisassemblyOnly) &&726 "Functions to disassemble should be only specified together with "727 "--show-disassembly-only");728 729 if (ShowDisassemblyOnly)730 outs() << "\nDisassembly of " << FileName << ":\n";731 732 // Dissassemble a text section.733 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();734 SI != SE; ++SI) {735 const SectionRef &Section = *SI;736 if (!Section.isText())737 continue;738 739 uint64_t ImageLoadAddr = getPreferredBaseAddress();740 uint64_t SectionAddress = Section.getAddress() - ImageLoadAddr;741 uint64_t SectSize = Section.getSize();742 if (!SectSize)743 continue;744 745 // Register the text section.746 TextSections.insert({SectionAddress, SectSize});747 748 StringRef SectionName = unwrapOrError(Section.getName(), FileName);749 750 if (ShowDisassemblyOnly) {751 outs() << "\nDisassembly of section " << SectionName;752 outs() << " [" << format("0x%" PRIx64, Section.getAddress()) << ", "753 << format("0x%" PRIx64, Section.getAddress() + SectSize)754 << "]:\n\n";755 }756 757 if (isa<ELFObjectFileBase>(Obj) && SectionName == ".plt")758 continue;759 760 // Get the section data.761 ArrayRef<uint8_t> Bytes =762 arrayRefFromStringRef(unwrapOrError(Section.getContents(), FileName));763 764 // Get the list of all the symbols in this section.765 SectionSymbolsTy &Symbols = AllSymbols[Section];766 767 // Disassemble symbol by symbol.768 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {769 if (!dissassembleSymbol(SI, Bytes, Symbols, Section))770 exitWithError("disassembling error", FileName);771 }772 }773 774 if (!AddrsWithInvalidInstruction.empty()) {775 if (ShowDetailedWarning) {776 for (auto &Addr : AddrsWithInvalidInstruction) {777 WithColor::warning()778 << "Invalid instructions at " << format("%8" PRIx64, Addr.first)779 << " - " << format("%8" PRIx64, Addr.second) << "\n";780 }781 }782 WithColor::warning() << "Found " << AddrsWithInvalidInstruction.size()783 << " invalid instructions\n";784 AddrsWithInvalidInstruction.clear();785 }786 787 // Dissassemble rodata section to check if FS discriminator symbol exists.788 checkUseFSDiscriminator(Obj, AllSymbols);789}790 791void ProfiledBinary::checkUseFSDiscriminator(792 const ObjectFile *Obj, std::map<SectionRef, SectionSymbolsTy> &AllSymbols) {793 const char *FSDiscriminatorVar = "__llvm_fs_discriminator__";794 for (section_iterator SI = Obj->section_begin(), SE = Obj->section_end();795 SI != SE; ++SI) {796 const SectionRef &Section = *SI;797 if (!Section.isData() || Section.getSize() == 0)798 continue;799 SectionSymbolsTy &Symbols = AllSymbols[Section];800 801 for (std::size_t SI = 0, SE = Symbols.size(); SI != SE; ++SI) {802 if (Symbols[SI].Name == FSDiscriminatorVar) {803 UseFSDiscriminator = true;804 return;805 }806 }807 }808}809 810void ProfiledBinary::populateSymbolAddressList(const ObjectFile *Obj) {811 // Create a mapping from virtual address to symbol GUID and the other way812 // around.813 StringRef FileName = Obj->getFileName();814 for (const SymbolRef &Symbol : Obj->symbols()) {815 const uint64_t Addr = unwrapOrError(Symbol.getAddress(), FileName);816 const StringRef Name = unwrapOrError(Symbol.getName(), FileName);817 uint64_t GUID = Function::getGUIDAssumingExternalLinkage(Name);818 SymbolStartAddrs[GUID] = Addr;819 StartAddrToSymMap.emplace(Addr, GUID);820 }821}822 823void ProfiledBinary::loadSymbolsFromDWARFUnit(DWARFUnit &CompilationUnit) {824 for (const auto &DieInfo : CompilationUnit.dies()) {825 llvm::DWARFDie Die(&CompilationUnit, &DieInfo);826 827 if (!Die.isSubprogramDIE())828 continue;829 auto Name = Die.getName(llvm::DINameKind::LinkageName);830 if (!Name)831 Name = Die.getName(llvm::DINameKind::ShortName);832 if (!Name)833 continue;834 835 auto RangesOrError = Die.getAddressRanges();836 if (!RangesOrError)837 continue;838 const DWARFAddressRangesVector &Ranges = RangesOrError.get();839 840 if (Ranges.empty())841 continue;842 843 // Different DWARF symbols can have same function name, search or create844 // BinaryFunction indexed by the name.845 auto Ret = BinaryFunctions.emplace(Name, BinaryFunction());846 auto &Func = Ret.first->second;847 if (Ret.second)848 Func.FuncName = Ret.first->first;849 850 for (const auto &Range : Ranges) {851 uint64_t StartAddress = Range.LowPC;852 uint64_t EndAddress = Range.HighPC;853 854 if (EndAddress <= StartAddress ||855 StartAddress < getPreferredBaseAddress())856 continue;857 858 // We may want to know all ranges for one function. Here group the859 // ranges and store them into BinaryFunction.860 Func.Ranges.emplace_back(StartAddress, EndAddress);861 862 auto R = StartAddrToFuncRangeMap.emplace(StartAddress, FuncRange());863 if (R.second) {864 FuncRange &FRange = R.first->second;865 FRange.Func = &Func;866 FRange.StartAddress = StartAddress;867 FRange.EndAddress = EndAddress;868 } else {869 AddrsWithMultipleSymbols.insert(StartAddress);870 if (ShowDetailedWarning)871 WithColor::warning()872 << "Duplicated symbol start address at "873 << format("%8" PRIx64, StartAddress) << " "874 << R.first->second.getFuncName() << " and " << Name << "\n";875 }876 }877 }878}879 880void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) {881 auto DebugContext = llvm::DWARFContext::create(882 Obj, DWARFContext::ProcessDebugRelocations::Process, nullptr, DWPPath);883 if (!DebugContext)884 exitWithError("Error creating the debug info context", Path);885 886 for (const auto &CompilationUnit : DebugContext->compile_units())887 loadSymbolsFromDWARFUnit(*CompilationUnit);888 889 // Handles DWO sections that can either be in .o, .dwo or .dwp files.890 uint32_t NumOfDWOMissing = 0;891 for (const auto &CompilationUnit : DebugContext->compile_units()) {892 DWARFUnit *const DwarfUnit = CompilationUnit.get();893 if (DwarfUnit->getDWOId()) {894 DWARFUnit *DWOCU = DwarfUnit->getNonSkeletonUnitDIE(false).getDwarfUnit();895 if (!DWOCU->isDWOUnit()) {896 NumOfDWOMissing++;897 if (ShowDetailedWarning) {898 std::string DWOName = dwarf::toString(899 DwarfUnit->getUnitDIE().find(900 {dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}),901 "");902 WithColor::warning() << "DWO debug information for " << DWOName903 << " was not loaded.\n";904 }905 continue;906 }907 loadSymbolsFromDWARFUnit(*DWOCU);908 }909 }910 911 if (NumOfDWOMissing)912 WithColor::warning()913 << " DWO debug information was not loaded for " << NumOfDWOMissing914 << " modules. Please check the .o, .dwo or .dwp path.\n";915 if (BinaryFunctions.empty())916 WithColor::warning() << "Loading of DWARF info completed, but no binary "917 "functions have been retrieved.\n";918 // Populate the hash binary function map for MD5 function name lookup. This919 // is done after BinaryFunctions are finalized.920 for (auto &BinaryFunction : BinaryFunctions) {921 HashBinaryFunctions[MD5Hash(StringRef(BinaryFunction.first))] =922 &BinaryFunction.second;923 }924 925 if (!AddrsWithMultipleSymbols.empty()) {926 WithColor::warning() << "Found " << AddrsWithMultipleSymbols.size()927 << " start addresses with multiple symbols\n";928 AddrsWithMultipleSymbols.clear();929 }930}931 932void ProfiledBinary::populateSymbolListFromDWARF(933 ProfileSymbolList &SymbolList) {934 for (auto &I : StartAddrToFuncRangeMap)935 SymbolList.add(I.second.getFuncName());936}937 938symbolize::LLVMSymbolizer::Options ProfiledBinary::getSymbolizerOpts() const {939 symbolize::LLVMSymbolizer::Options SymbolizerOpts;940 SymbolizerOpts.PrintFunctions =941 DILineInfoSpecifier::FunctionNameKind::LinkageName;942 SymbolizerOpts.Demangle = false;943 SymbolizerOpts.DefaultArch = TheTriple.getArchName().str();944 SymbolizerOpts.UseSymbolTable = false;945 SymbolizerOpts.RelativeAddresses = false;946 SymbolizerOpts.DWPName = DWPPath;947 return SymbolizerOpts;948}949 950SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,951 bool UseCanonicalFnName,952 bool UseProbeDiscriminator) {953 assert(this == IP.Binary &&954 "Binary should only symbolize its own instruction");955 DIInliningInfo InlineStack =956 unwrapOrError(Symbolizer->symbolizeInlinedCode(957 SymbolizerPath.str(), getSectionedAddress(IP.Address)),958 SymbolizerPath);959 960 SampleContextFrameVector CallStack;961 for (int32_t I = InlineStack.getNumberOfFrames() - 1; I >= 0; I--) {962 const auto &CallerFrame = InlineStack.getFrame(I);963 if (CallerFrame.FunctionName.empty() ||964 (CallerFrame.FunctionName == "<invalid>"))965 break;966 967 StringRef FunctionName(CallerFrame.FunctionName);968 if (UseCanonicalFnName)969 FunctionName = FunctionSamples::getCanonicalFnName(FunctionName);970 971 uint32_t Discriminator = CallerFrame.Discriminator;972 uint32_t LineOffset = (CallerFrame.Line - CallerFrame.StartLine) & 0xffff;973 if (UseProbeDiscriminator) {974 LineOffset =975 PseudoProbeDwarfDiscriminator::extractProbeIndex(Discriminator);976 Discriminator = 0;977 }978 979 LineLocation Line(LineOffset, Discriminator);980 auto It = NameStrings.insert(FunctionName.str());981 CallStack.emplace_back(FunctionId(StringRef(*It.first)), Line);982 }983 984 return CallStack;985}986 987StringRef ProfiledBinary::symbolizeDataAddress(uint64_t Address) {988 DIGlobal DataDIGlobal =989 unwrapOrError(Symbolizer->symbolizeData(SymbolizerPath.str(),990 getSectionedAddress(Address)),991 SymbolizerPath);992 decltype(NameStrings)::iterator Iter;993 std::tie(Iter, std::ignore) = NameStrings.insert(DataDIGlobal.Name);994 return StringRef(*Iter);995}996 997void ProfiledBinary::computeInlinedContextSizeForRange(uint64_t RangeBegin,998 uint64_t RangeEnd) {999 InstructionPointer IP(this, RangeBegin, true);1000 1001 if (IP.Address != RangeBegin)1002 WithColor::warning() << "Invalid start instruction at "1003 << format("%8" PRIx64, RangeBegin) << "\n";1004 1005 if (IP.Address >= RangeEnd)1006 return;1007 1008 do {1009 const SampleContextFrameVector SymbolizedCallStack =1010 getFrameLocationStack(IP.Address, UsePseudoProbes);1011 uint64_t Size = AddressToInstSizeMap[IP.Address];1012 // Record instruction size for the corresponding context1013 FuncSizeTracker.addInstructionForContext(SymbolizedCallStack, Size);1014 1015 } while (IP.advance() && IP.Address < RangeEnd);1016}1017 1018void ProfiledBinary::computeInlinedContextSizeForFunc(1019 const BinaryFunction *Func) {1020 // Note that a function can be spilt into multiple ranges, so compute for all1021 // ranges of the function.1022 for (const auto &Range : Func->Ranges)1023 computeInlinedContextSizeForRange(Range.first, Range.second);1024 1025 // Track optimized-away inlinee for probed binary. A function inlined and then1026 // optimized away should still have their probes left over in places.1027 if (usePseudoProbes()) {1028 auto I = TopLevelProbeFrameMap.find(Func->FuncName);1029 if (I != TopLevelProbeFrameMap.end()) {1030 BinarySizeContextTracker::ProbeFrameStack ProbeContext;1031 FuncSizeTracker.trackInlineesOptimizedAway(ProbeDecoder, *I->second,1032 ProbeContext);1033 }1034 }1035}1036 1037void ProfiledBinary::inferMissingFrames(1038 const SmallVectorImpl<uint64_t> &Context,1039 SmallVectorImpl<uint64_t> &NewContext) {1040 MissingContextInferrer->inferMissingFrames(Context, NewContext);1041}1042 1043InstructionPointer::InstructionPointer(const ProfiledBinary *Binary,1044 uint64_t Address, bool RoundToNext)1045 : Binary(Binary), Address(Address) {1046 Index = Binary->getIndexForAddr(Address);1047 if (RoundToNext) {1048 // we might get address which is not the code1049 // it should round to the next valid address1050 if (Index >= Binary->getCodeAddrVecSize())1051 this->Address = UINT64_MAX;1052 else1053 this->Address = Binary->getAddressforIndex(Index);1054 }1055}1056 1057bool InstructionPointer::advance() {1058 Index++;1059 if (Index >= Binary->getCodeAddrVecSize()) {1060 Address = UINT64_MAX;1061 return false;1062 }1063 Address = Binary->getAddressforIndex(Index);1064 return true;1065}1066 1067bool InstructionPointer::backward() {1068 if (Index == 0) {1069 Address = 0;1070 return false;1071 }1072 Index--;1073 Address = Binary->getAddressforIndex(Index);1074 return true;1075}1076 1077void InstructionPointer::update(uint64_t Addr) {1078 Address = Addr;1079 Index = Binary->getIndexForAddr(Address);1080}1081 1082} // end namespace sampleprof1083} // end namespace llvm1084