742 lines · cpp
1//===- lib/MC/MCPseudoProbe.cpp - Pseudo probe encoding support ----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/MC/MCPseudoProbe.h"10#include "llvm/ADT/STLExtras.h"11#include "llvm/IR/PseudoProbe.h"12#include "llvm/MC/MCAsmInfo.h"13#include "llvm/MC/MCAssembler.h"14#include "llvm/MC/MCContext.h"15#include "llvm/MC/MCExpr.h"16#include "llvm/MC/MCObjectFileInfo.h"17#include "llvm/MC/MCObjectStreamer.h"18#include "llvm/MC/MCSymbol.h"19#include "llvm/Support/Endian.h"20#include "llvm/Support/Error.h"21#include "llvm/Support/LEB128.h"22#include "llvm/Support/MD5.h"23#include "llvm/Support/raw_ostream.h"24#include <algorithm>25#include <cassert>26#include <limits>27#include <sstream>28#include <vector>29 30#define DEBUG_TYPE "mcpseudoprobe"31 32using namespace llvm;33using namespace support;34 35#ifndef NDEBUG36int MCPseudoProbeTable::DdgPrintIndent = 0;37#endif38 39static const MCExpr *buildSymbolDiff(MCObjectStreamer *MCOS, const MCSymbol *A,40 const MCSymbol *B) {41 MCContext &Context = MCOS->getContext();42 const MCExpr *ARef = MCSymbolRefExpr::create(A, Context);43 const MCExpr *BRef = MCSymbolRefExpr::create(B, Context);44 const MCExpr *AddrDelta =45 MCBinaryExpr::create(MCBinaryExpr::Sub, ARef, BRef, Context);46 return AddrDelta;47}48 49uint64_t MCDecodedPseudoProbe::getGuid() const { return InlineTree->Guid; }50 51void MCPseudoProbe::emit(MCObjectStreamer *MCOS,52 const MCPseudoProbe *LastProbe) const {53 bool IsSentinel = isSentinelProbe(getAttributes());54 assert((LastProbe || IsSentinel) &&55 "Last probe should not be null for non-sentinel probes");56 57 // Emit Index58 MCOS->emitULEB128IntValue(Index);59 // Emit Type and the flag:60 // Type (bit 0 to 3), with bit 4 to 6 for attributes.61 // Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether62 // the following field is a symbolic code address or an address delta.63 // Emit FS discriminator64 assert(Type <= 0xF && "Probe type too big to encode, exceeding 15");65 auto NewAttributes = Attributes;66 if (Discriminator)67 NewAttributes |= (uint32_t)PseudoProbeAttributes::HasDiscriminator;68 assert(NewAttributes <= 0x7 &&69 "Probe attributes too big to encode, exceeding 7");70 uint8_t PackedType = Type | (NewAttributes << 4);71 uint8_t Flag =72 !IsSentinel ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0;73 MCOS->emitInt8(Flag | PackedType);74 75 if (!IsSentinel) {76 // Emit the delta between the address label and LastProbe.77 const MCExpr *AddrDelta =78 buildSymbolDiff(MCOS, Label, LastProbe->getLabel());79 int64_t Delta;80 if (AddrDelta->evaluateAsAbsolute(Delta, MCOS->getAssemblerPtr())) {81 MCOS->emitSLEB128IntValue(Delta);82 } else {83 auto *F = MCOS->getCurrentFragment();84 F->makeLEB(true, AddrDelta);85 MCOS->newFragment();86 }87 } else {88 // Emit the GUID of the split function that the sentinel probe represents.89 MCOS->emitInt64(Guid);90 }91 92 if (Discriminator)93 MCOS->emitULEB128IntValue(Discriminator);94 95 LLVM_DEBUG({96 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);97 dbgs() << "Probe: " << Index << "\n";98 });99}100 101void MCPseudoProbeInlineTree::addPseudoProbe(102 const MCPseudoProbe &Probe, const MCPseudoProbeInlineStack &InlineStack) {103 // The function should not be called on the root.104 assert(isRoot() && "Should only be called on root");105 106 // When it comes here, the input look like:107 // Probe: GUID of C, ...108 // InlineStack: [88, A], [66, B]109 // which means, Function A inlines function B at call site with a probe id of110 // 88, and B inlines C at probe 66. The tri-tree expects a tree path like {[0,111 // A], [88, B], [66, C]} to locate the tree node where the probe should be112 // added. Note that the edge [0, A] means A is the top-level function we are113 // emitting probes for.114 115 // Make a [0, A] edge.116 // An empty inline stack means the function that the probe originates from117 // is a top-level function.118 InlineSite Top;119 if (InlineStack.empty()) {120 Top = InlineSite(Probe.getGuid(), 0);121 } else {122 Top = InlineSite(std::get<0>(InlineStack.front()), 0);123 }124 125 auto *Cur = getOrAddNode(Top);126 127 // Make interior edges by walking the inline stack. Once it's done, Cur should128 // point to the node that the probe originates from.129 if (!InlineStack.empty()) {130 auto Iter = InlineStack.begin();131 auto Index = std::get<1>(*Iter);132 Iter++;133 for (; Iter != InlineStack.end(); Iter++) {134 // Make an edge by using the previous probe id and current GUID.135 Cur = Cur->getOrAddNode(InlineSite(std::get<0>(*Iter), Index));136 Index = std::get<1>(*Iter);137 }138 Cur = Cur->getOrAddNode(InlineSite(Probe.getGuid(), Index));139 }140 141 Cur->Probes.push_back(Probe);142}143 144void MCPseudoProbeInlineTree::emit(MCObjectStreamer *MCOS,145 const MCPseudoProbe *&LastProbe) {146 LLVM_DEBUG({147 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);148 dbgs() << "Group [\n";149 MCPseudoProbeTable::DdgPrintIndent += 2;150 });151 assert(!isRoot() && "Root should be handled separately");152 153 // Emit probes grouped by GUID.154 LLVM_DEBUG({155 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);156 dbgs() << "GUID: " << Guid << "\n";157 });158 // Emit Guid159 MCOS->emitInt64(Guid);160 // Emit number of probes in this node, including a sentinel probe for161 // top-level functions if needed.162 bool NeedSentinel = false;163 if (Parent->isRoot()) {164 assert(isSentinelProbe(LastProbe->getAttributes()) &&165 "Starting probe of a top-level function should be a sentinel probe");166 // The main body of a split function doesn't need a sentinel probe.167 if (LastProbe->getGuid() != Guid)168 NeedSentinel = true;169 }170 171 MCOS->emitULEB128IntValue(Probes.size() + NeedSentinel);172 // Emit number of direct inlinees173 MCOS->emitULEB128IntValue(Children.size());174 // Emit sentinel probe for top-level functions175 if (NeedSentinel)176 LastProbe->emit(MCOS, nullptr);177 178 // Emit probes in this group179 for (const auto &Probe : Probes) {180 Probe.emit(MCOS, LastProbe);181 LastProbe = &Probe;182 }183 184 // Emit sorted descendant. InlineSite is unique for each pair, so there will185 // be no ordering of Inlinee based on MCPseudoProbeInlineTree*186 using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>;187 std::vector<InlineeType> Inlinees;188 for (const auto &Child : Children)189 Inlinees.emplace_back(Child.first, Child.second.get());190 llvm::sort(Inlinees, llvm::less_first());191 192 for (const auto &Inlinee : Inlinees) {193 // Emit probe index194 MCOS->emitULEB128IntValue(std::get<1>(Inlinee.first));195 LLVM_DEBUG({196 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);197 dbgs() << "InlineSite: " << std::get<1>(Inlinee.first) << "\n";198 });199 // Emit the group200 Inlinee.second->emit(MCOS, LastProbe);201 }202 203 LLVM_DEBUG({204 MCPseudoProbeTable::DdgPrintIndent -= 2;205 dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);206 dbgs() << "]\n";207 });208}209 210void MCPseudoProbeSections::emit(MCObjectStreamer *MCOS) {211 MCContext &Ctx = MCOS->getContext();212 SmallVector<std::pair<MCSymbol *, MCPseudoProbeInlineTree *>> Vec;213 Vec.reserve(MCProbeDivisions.size());214 for (auto &ProbeSec : MCProbeDivisions)215 Vec.emplace_back(ProbeSec.first, &ProbeSec.second);216 for (auto I : llvm::enumerate(MCOS->getAssembler()))217 I.value().setOrdinal(I.index());218 llvm::sort(Vec, [](auto A, auto B) {219 return A.first->getSection().getOrdinal() <220 B.first->getSection().getOrdinal();221 });222 for (auto [FuncSym, RootPtr] : Vec) {223 const auto &Root = *RootPtr;224 if (auto *S = Ctx.getObjectFileInfo()->getPseudoProbeSection(225 FuncSym->getSection())) {226 // Switch to the .pseudoprobe section or a comdat group.227 MCOS->switchSection(S);228 // Emit probes grouped by GUID.229 // Emit sorted descendant. InlineSite is unique for each pair, so there230 // will be no ordering of Inlinee based on MCPseudoProbeInlineTree*231 using InlineeType = std::pair<InlineSite, MCPseudoProbeInlineTree *>;232 std::vector<InlineeType> Inlinees;233 for (const auto &Child : Root.getChildren())234 Inlinees.emplace_back(Child.first, Child.second.get());235 llvm::sort(Inlinees, llvm::less_first());236 237 for (const auto &Inlinee : Inlinees) {238 // Emit the group guarded by a sentinel probe.239 MCPseudoProbe SentinelProbe(240 const_cast<MCSymbol *>(FuncSym), MD5Hash(FuncSym->getName()),241 (uint32_t)PseudoProbeReservedId::Invalid,242 (uint32_t)PseudoProbeType::Block,243 (uint32_t)PseudoProbeAttributes::Sentinel, 0);244 const MCPseudoProbe *Probe = &SentinelProbe;245 Inlinee.second->emit(MCOS, Probe);246 }247 }248 }249}250 251//252// This emits the pseudo probe tables.253//254void MCPseudoProbeTable::emit(MCObjectStreamer *MCOS) {255 MCContext &Ctx = MCOS->getContext();256 auto &ProbeTable = Ctx.getMCPseudoProbeTable();257 258 // Bail out early so we don't switch to the pseudo_probe section needlessly259 // and in doing so create an unnecessary (if empty) section.260 auto &ProbeSections = ProbeTable.getProbeSections();261 if (ProbeSections.empty())262 return;263 264 LLVM_DEBUG(MCPseudoProbeTable::DdgPrintIndent = 0);265 266 // Put out the probe.267 ProbeSections.emit(MCOS);268}269 270static StringRef getProbeFNameForGUID(const GUIDProbeFunctionMap &GUID2FuncMAP,271 uint64_t GUID) {272 auto It = GUID2FuncMAP.find(GUID);273 assert(It != GUID2FuncMAP.end() &&274 "Probe function must exist for a valid GUID");275 return It->FuncName;276}277 278void MCPseudoProbeFuncDesc::print(raw_ostream &OS) {279 OS << "GUID: " << FuncGUID << " Name: " << FuncName << "\n";280 OS << "Hash: " << FuncHash << "\n";281}282 283void MCDecodedPseudoProbe::getInlineContext(284 SmallVectorImpl<MCPseudoProbeFrameLocation> &ContextStack,285 const GUIDProbeFunctionMap &GUID2FuncMAP) const {286 uint32_t Begin = ContextStack.size();287 MCDecodedPseudoProbeInlineTree *Cur = InlineTree;288 // It will add the string of each node's inline site during iteration.289 // Note that it won't include the probe's belonging function(leaf location)290 while (Cur->hasInlineSite()) {291 StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, Cur->Parent->Guid);292 ContextStack.emplace_back(MCPseudoProbeFrameLocation(293 FuncName, std::get<1>(Cur->getInlineSite())));294 Cur = static_cast<MCDecodedPseudoProbeInlineTree *>(Cur->Parent);295 }296 // Make the ContextStack in caller-callee order297 std::reverse(ContextStack.begin() + Begin, ContextStack.end());298}299 300std::string MCDecodedPseudoProbe::getInlineContextStr(301 const GUIDProbeFunctionMap &GUID2FuncMAP) const {302 std::ostringstream OContextStr;303 SmallVector<MCPseudoProbeFrameLocation, 16> ContextStack;304 getInlineContext(ContextStack, GUID2FuncMAP);305 for (auto &Cxt : ContextStack) {306 if (OContextStr.str().size())307 OContextStr << " @ ";308 OContextStr << Cxt.first.str() << ":" << Cxt.second;309 }310 return OContextStr.str();311}312 313static const char *PseudoProbeTypeStr[3] = {"Block", "IndirectCall",314 "DirectCall"};315 316void MCDecodedPseudoProbe::print(raw_ostream &OS,317 const GUIDProbeFunctionMap &GUID2FuncMAP,318 bool ShowName) const {319 OS << "FUNC: ";320 if (ShowName) {321 StringRef FuncName = getProbeFNameForGUID(GUID2FuncMAP, getGuid());322 OS << FuncName.str() << " ";323 } else {324 OS << getGuid() << " ";325 }326 OS << "Index: " << Index << " ";327 if (Discriminator)328 OS << "Discriminator: " << Discriminator << " ";329 OS << "Type: " << PseudoProbeTypeStr[static_cast<uint8_t>(Type)] << " ";330 std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP);331 if (InlineContextStr.size()) {332 OS << "Inlined: @ ";333 OS << InlineContextStr;334 }335 OS << "\n";336}337 338template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnencodedNumber() {339 if (Data + sizeof(T) > End) {340 return std::error_code();341 }342 T Val = endian::readNext<T, llvm::endianness::little>(Data);343 return ErrorOr<T>(Val);344}345 346template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readUnsignedNumber() {347 unsigned NumBytesRead = 0;348 uint64_t Val = decodeULEB128(Data, &NumBytesRead);349 if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) {350 return std::error_code();351 }352 Data += NumBytesRead;353 return ErrorOr<T>(static_cast<T>(Val));354}355 356template <typename T> ErrorOr<T> MCPseudoProbeDecoder::readSignedNumber() {357 unsigned NumBytesRead = 0;358 int64_t Val = decodeSLEB128(Data, &NumBytesRead);359 if (Val > std::numeric_limits<T>::max() || (Data + NumBytesRead > End)) {360 return std::error_code();361 }362 Data += NumBytesRead;363 return ErrorOr<T>(static_cast<T>(Val));364}365 366ErrorOr<StringRef> MCPseudoProbeDecoder::readString(uint32_t Size) {367 StringRef Str(reinterpret_cast<const char *>(Data), Size);368 if (Data + Size > End) {369 return std::error_code();370 }371 Data += Size;372 return ErrorOr<StringRef>(Str);373}374 375bool MCPseudoProbeDecoder::buildGUID2FuncDescMap(const uint8_t *Start,376 std::size_t Size,377 bool IsMMapped) {378 // The pseudo_probe_desc section has a format like:379 // .section .pseudo_probe_desc,"",@progbits380 // .quad -5182264717993193164 // GUID381 // .quad 4294967295 // Hash382 // .uleb 3 // Name size383 // .ascii "foo" // Name384 // .quad -2624081020897602054385 // .quad 174696971957386 // .uleb 34387 // .ascii "main"388 389 Data = Start;390 End = Data + Size;391 392 uint32_t FuncDescCount = 0;393 while (Data < End) {394 // GUID395 if (!readUnencodedNumber<uint64_t>())396 return false;397 // Hash398 if (!readUnencodedNumber<uint64_t>())399 return false;400 401 auto ErrorOrNameSize = readUnsignedNumber<uint32_t>();402 if (!ErrorOrNameSize)403 return false;404 // Function name405 if (!readString(*ErrorOrNameSize))406 return false;407 ++FuncDescCount;408 }409 assert(Data == End && "Have unprocessed data in pseudo_probe_desc section");410 GUID2FuncDescMap.reserve(FuncDescCount);411 412 Data = Start;413 End = Data + Size;414 while (Data < End) {415 uint64_t GUID =416 cantFail(errorOrToExpected(readUnencodedNumber<uint64_t>()));417 uint64_t Hash =418 cantFail(errorOrToExpected(readUnencodedNumber<uint64_t>()));419 uint32_t NameSize =420 cantFail(errorOrToExpected(readUnsignedNumber<uint32_t>()));421 StringRef Name = cantFail(errorOrToExpected(readString(NameSize)));422 423 // Initialize PseudoProbeFuncDesc and populate it into GUID2FuncDescMap424 GUID2FuncDescMap.emplace_back(425 GUID, Hash, IsMMapped ? Name : Name.copy(FuncNameAllocator));426 }427 assert(Data == End && "Have unprocessed data in pseudo_probe_desc section");428 assert(GUID2FuncDescMap.size() == FuncDescCount &&429 "Mismatching function description count pre- and post-parsing");430 llvm::sort(GUID2FuncDescMap, [](const auto &LHS, const auto &RHS) {431 return LHS.FuncGUID < RHS.FuncGUID;432 });433 return true;434}435 436template <bool IsTopLevelFunc>437bool MCPseudoProbeDecoder::buildAddress2ProbeMap(438 MCDecodedPseudoProbeInlineTree *Cur, uint64_t &LastAddr,439 const Uint64Set &GuidFilter, const Uint64Map &FuncStartAddrs,440 const uint32_t CurChildIndex) {441 // The pseudo_probe section encodes an inline forest and each tree has a442 // format defined in MCPseudoProbe.h443 444 uint32_t Index = 0;445 if (IsTopLevelFunc) {446 // Use a sequential id for top level inliner.447 Index = CurChildIndex;448 } else {449 // Read inline site for inlinees450 Index = cantFail(errorOrToExpected(readUnsignedNumber<uint32_t>()));451 }452 453 // Read guid454 uint64_t Guid = cantFail(errorOrToExpected(readUnencodedNumber<uint64_t>()));455 456 // Decide if top-level node should be disgarded.457 if (IsTopLevelFunc && !GuidFilter.empty() && !GuidFilter.count(Guid))458 Cur = nullptr;459 460 // If the incoming node is null, all its children nodes should be disgarded.461 if (Cur) {462 // Switch/add to a new tree node(inlinee)463 Cur->getChildren()[CurChildIndex] =464 MCDecodedPseudoProbeInlineTree(InlineSite(Guid, Index), Cur);465 Cur = &Cur->getChildren()[CurChildIndex];466 if (IsTopLevelFunc && !EncodingIsAddrBased) {467 if (auto V = FuncStartAddrs.lookup(Guid))468 LastAddr = V;469 }470 }471 472 // Read number of probes in the current node.473 uint32_t NodeCount =474 cantFail(errorOrToExpected(readUnsignedNumber<uint32_t>()));475 uint32_t CurrentProbeCount = 0;476 // Read number of direct inlinees477 uint32_t ChildrenToProcess =478 cantFail(errorOrToExpected(readUnsignedNumber<uint32_t>()));479 // Read all probes in this node480 for (std::size_t I = 0; I < NodeCount; I++) {481 // Read index482 uint32_t Index =483 cantFail(errorOrToExpected(readUnsignedNumber<uint32_t>()));484 // Read type | flag.485 uint8_t Value = cantFail(errorOrToExpected(readUnencodedNumber<uint8_t>()));486 uint8_t Kind = Value & 0xf;487 uint8_t Attr = (Value & 0x70) >> 4;488 // Read address489 uint64_t Addr = 0;490 if (Value & 0x80) {491 int64_t Offset = cantFail(errorOrToExpected(readSignedNumber<int64_t>()));492 Addr = LastAddr + Offset;493 } else {494 Addr = cantFail(errorOrToExpected(readUnencodedNumber<int64_t>()));495 if (isSentinelProbe(Attr)) {496 // For sentinel probe, the addr field actually stores the GUID of the497 // split function. Convert it to the real address.498 if (auto V = FuncStartAddrs.lookup(Addr))499 Addr = V;500 } else {501 // For now we assume all probe encoding should be either based on502 // leading probe address or function start address.503 // The scheme is for downwards compatibility.504 // TODO: retire this scheme once compatibility is no longer an issue.505 EncodingIsAddrBased = true;506 }507 }508 509 uint32_t Discriminator = 0;510 if (hasDiscriminator(Attr)) {511 Discriminator =512 cantFail(errorOrToExpected(readUnsignedNumber<uint32_t>()));513 }514 515 if (Cur && !isSentinelProbe(Attr)) {516 PseudoProbeVec.emplace_back(Addr, Index, PseudoProbeType(Kind), Attr,517 Discriminator, Cur);518 ++CurrentProbeCount;519 }520 LastAddr = Addr;521 }522 523 if (Cur) {524 Cur->setProbes(525 MutableArrayRef(PseudoProbeVec).take_back(CurrentProbeCount));526 InlineTreeVec.resize(InlineTreeVec.size() + ChildrenToProcess);527 Cur->getChildren() =528 MutableArrayRef(InlineTreeVec).take_back(ChildrenToProcess);529 }530 for (uint32_t I = 0; I < ChildrenToProcess; I++) {531 buildAddress2ProbeMap<false>(Cur, LastAddr, GuidFilter, FuncStartAddrs, I);532 }533 return Cur;534}535 536template <bool IsTopLevelFunc>537bool MCPseudoProbeDecoder::countRecords(bool &Discard, uint32_t &ProbeCount,538 uint32_t &InlinedCount,539 const Uint64Set &GuidFilter) {540 if (!IsTopLevelFunc)541 // Read inline site for inlinees542 if (!readUnsignedNumber<uint32_t>())543 return false;544 545 // Read guid546 auto ErrorOrCurGuid = readUnencodedNumber<uint64_t>();547 if (!ErrorOrCurGuid)548 return false;549 uint64_t Guid = std::move(*ErrorOrCurGuid);550 551 // Decide if top-level node should be disgarded.552 if (IsTopLevelFunc) {553 Discard = !GuidFilter.empty() && !GuidFilter.count(Guid);554 if (!Discard)555 // Allocate an entry for top-level function record.556 ++InlinedCount;557 }558 559 // Read number of probes in the current node.560 auto ErrorOrNodeCount = readUnsignedNumber<uint32_t>();561 if (!ErrorOrNodeCount)562 return false;563 uint32_t NodeCount = std::move(*ErrorOrNodeCount);564 uint32_t CurrentProbeCount = 0;565 566 // Read number of direct inlinees567 auto ErrorOrCurChildrenToProcess = readUnsignedNumber<uint32_t>();568 if (!ErrorOrCurChildrenToProcess)569 return false;570 uint32_t ChildrenToProcess = std::move(*ErrorOrCurChildrenToProcess);571 572 // Read all probes in this node573 for (std::size_t I = 0; I < NodeCount; I++) {574 // Read index575 if (!readUnsignedNumber<uint32_t>())576 return false;577 578 // Read type | flag.579 auto ErrorOrValue = readUnencodedNumber<uint8_t>();580 if (!ErrorOrValue)581 return false;582 uint8_t Value = std::move(*ErrorOrValue);583 584 uint8_t Attr = (Value & 0x70) >> 4;585 if (Value & 0x80) {586 // Offset587 if (!readSignedNumber<int64_t>())588 return false;589 } else {590 // Addr591 if (!readUnencodedNumber<int64_t>())592 return false;593 }594 595 if (hasDiscriminator(Attr))596 // Discriminator597 if (!readUnsignedNumber<uint32_t>())598 return false;599 600 if (!Discard && !isSentinelProbe(Attr))601 ++CurrentProbeCount;602 }603 604 if (!Discard) {605 ProbeCount += CurrentProbeCount;606 InlinedCount += ChildrenToProcess;607 }608 609 for (uint32_t I = 0; I < ChildrenToProcess; I++)610 if (!countRecords<false>(Discard, ProbeCount, InlinedCount, GuidFilter))611 return false;612 return true;613}614 615bool MCPseudoProbeDecoder::buildAddress2ProbeMap(616 const uint8_t *Start, std::size_t Size, const Uint64Set &GuidFilter,617 const Uint64Map &FuncStartAddrs) {618 // For function records in the order of their appearance in the encoded data619 // (DFS), count the number of contained probes and inlined function records.620 uint32_t ProbeCount = 0;621 uint32_t InlinedCount = 0;622 uint32_t TopLevelFuncs = 0;623 Data = Start;624 End = Data + Size;625 bool Discard = false;626 while (Data < End) {627 if (!countRecords<true>(Discard, ProbeCount, InlinedCount, GuidFilter))628 return false;629 TopLevelFuncs += !Discard;630 }631 assert(Data == End && "Have unprocessed data in pseudo_probe section");632 PseudoProbeVec.reserve(ProbeCount);633 InlineTreeVec.reserve(InlinedCount);634 635 // Allocate top-level function records as children of DummyInlineRoot.636 InlineTreeVec.resize(TopLevelFuncs);637 DummyInlineRoot.getChildren() = MutableArrayRef(InlineTreeVec);638 639 Data = Start;640 End = Data + Size;641 uint64_t LastAddr = 0;642 uint32_t CurChildIndex = 0;643 while (Data < End)644 CurChildIndex += buildAddress2ProbeMap<true>(645 &DummyInlineRoot, LastAddr, GuidFilter, FuncStartAddrs, CurChildIndex);646 assert(Data == End && "Have unprocessed data in pseudo_probe section");647 assert(PseudoProbeVec.size() == ProbeCount &&648 "Mismatching probe count pre- and post-parsing");649 assert(InlineTreeVec.size() == InlinedCount &&650 "Mismatching function records count pre- and post-parsing");651 652 std::vector<std::pair<uint64_t, uint32_t>> SortedA2P(ProbeCount);653 for (const auto &[I, Probe] : llvm::enumerate(PseudoProbeVec))654 SortedA2P[I] = {Probe.getAddress(), I};655 llvm::sort(SortedA2P);656 Address2ProbesMap.reserve(ProbeCount);657 for (const uint32_t I : llvm::make_second_range(SortedA2P))658 Address2ProbesMap.emplace_back(PseudoProbeVec[I]);659 SortedA2P.clear();660 return true;661}662 663void MCPseudoProbeDecoder::printGUID2FuncDescMap(raw_ostream &OS) {664 OS << "Pseudo Probe Desc:\n";665 for (auto &I : GUID2FuncDescMap)666 I.print(OS);667}668 669void MCPseudoProbeDecoder::printProbeForAddress(raw_ostream &OS,670 uint64_t Address) {671 for (const MCDecodedPseudoProbe &Probe : Address2ProbesMap.find(Address)) {672 OS << " [Probe]:\t";673 Probe.print(OS, GUID2FuncDescMap, true);674 }675}676 677void MCPseudoProbeDecoder::printProbesForAllAddresses(raw_ostream &OS) {678 uint64_t PrevAddress = INT64_MAX;679 for (MCDecodedPseudoProbe &Probe : Address2ProbesMap) {680 uint64_t Address = Probe.getAddress();681 if (Address != PrevAddress) {682 PrevAddress = Address;683 OS << "Address:\t" << Address << '\n';684 }685 OS << " [Probe]:\t";686 Probe.print(OS, GUID2FuncDescMap, true);687 }688}689 690const MCDecodedPseudoProbe *691MCPseudoProbeDecoder::getCallProbeForAddr(uint64_t Address) const {692 const MCDecodedPseudoProbe *CallProbe = nullptr;693 for (const MCDecodedPseudoProbe &Probe : Address2ProbesMap.find(Address)) {694 if (Probe.isCall()) {695 // Disabling the assert and returning first call probe seen so far.696 // Subsequent call probes, if any, are ignored. Due to the the way697 // .pseudo_probe section is decoded, probes of the same-named independent698 // static functions are merged thus multiple call probes may be seen for a699 // callsite. This should only happen to compiler-generated statics, with700 // -funique-internal-linkage-names where user statics get unique names.701 //702 // TODO: re-enable or narrow down the assert to static functions only.703 //704 // assert(!CallProbe &&705 // "There should be only one call probe corresponding to address "706 // "which is a callsite.");707 CallProbe = &Probe;708 break;709 }710 }711 return CallProbe;712}713 714const MCPseudoProbeFuncDesc *715MCPseudoProbeDecoder::getFuncDescForGUID(uint64_t GUID) const {716 auto It = GUID2FuncDescMap.find(GUID);717 assert(It != GUID2FuncDescMap.end() && "Function descriptor doesn't exist");718 return &*It;719}720 721void MCPseudoProbeDecoder::getInlineContextForProbe(722 const MCDecodedPseudoProbe *Probe,723 SmallVectorImpl<MCPseudoProbeFrameLocation> &InlineContextStack,724 bool IncludeLeaf) const {725 Probe->getInlineContext(InlineContextStack, GUID2FuncDescMap);726 if (!IncludeLeaf)727 return;728 // Note that the context from probe doesn't include leaf frame,729 // hence we need to retrieve and prepend leaf if requested.730 const auto *FuncDesc = getFuncDescForGUID(Probe->getGuid());731 InlineContextStack.emplace_back(732 MCPseudoProbeFrameLocation(FuncDesc->FuncName, Probe->getIndex()));733}734 735const MCPseudoProbeFuncDesc *MCPseudoProbeDecoder::getInlinerDescForProbe(736 const MCDecodedPseudoProbe *Probe) const {737 MCDecodedPseudoProbeInlineTree *InlinerNode = Probe->getInlineTreeNode();738 if (!InlinerNode->hasInlineSite())739 return nullptr;740 return getFuncDescForGUID(InlinerNode->Parent->Guid);741}742