brintos / llvm-project-archived public Read only

1//===- bolt/Profile/DataAggregator.cpp - Perf data aggregator -------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This family of functions reads profile data written by perf record,10// aggregate it and then write it back to an output file.11//12//===----------------------------------------------------------------------===//13 14#include "bolt/Profile/DataAggregator.h"15#include "bolt/Core/BinaryContext.h"16#include "bolt/Core/BinaryFunction.h"17#include "bolt/Passes/BinaryPasses.h"18#include "bolt/Profile/BoltAddressTranslation.h"19#include "bolt/Profile/Heatmap.h"20#include "bolt/Profile/YAMLProfileWriter.h"21#include "bolt/Utils/CommandLineOpts.h"22#include "bolt/Utils/Utils.h"23#include "llvm/ADT/STLExtras.h"24#include "llvm/ADT/ScopeExit.h"25#include "llvm/Support/CommandLine.h"26#include "llvm/Support/Compiler.h"27#include "llvm/Support/Debug.h"28#include "llvm/Support/Errc.h"29#include "llvm/Support/FileSystem.h"30#include "llvm/Support/Process.h"31#include "llvm/Support/Program.h"32#include "llvm/Support/Regex.h"33#include "llvm/Support/Timer.h"34#include "llvm/Support/raw_ostream.h"35#include <map>36#include <optional>37#include <unordered_map>38#include <utility>39 40#define DEBUG_TYPE "aggregator"41 42using namespace llvm;43using namespace bolt;44 45namespace opts {46 47static cl::opt<bool>48    BasicAggregation("basic-events",49                     cl::desc("aggregate basic events (without brstack info)"),50                     cl::cat(AggregatorCategory));51 52static cl::alias BasicAggregationAlias("ba",53                                       cl::desc("Alias for --basic-events"),54                                       cl::aliasopt(BasicAggregation));55 56static cl::opt<bool> DeprecatedBasicAggregationNl(57    "nl", cl::desc("Alias for --basic-events (deprecated. Use --ba)"),58    cl::cat(AggregatorCategory), cl::ReallyHidden,59    cl::callback([](const bool &Enabled) {60      errs()61          << "BOLT-WARNING: '-nl' is deprecated, please use '--ba' instead.\n";62      BasicAggregation = Enabled;63    }));64 65cl::opt<bool> ArmSPE("spe", cl::desc("Enable Arm SPE mode."),66                     cl::cat(AggregatorCategory));67 68static cl::opt<std::string> ITraceAggregation(69    "itrace", cl::desc("Generate brstack info with perf itrace argument"),70    cl::cat(AggregatorCategory));71 72static cl::opt<bool>73FilterMemProfile("filter-mem-profile",74  cl::desc("if processing a memory profile, filter out stack or heap accesses "75           "that won't be useful for BOLT to reduce profile file size"),76  cl::init(true),77  cl::cat(AggregatorCategory));78 79static cl::opt<bool> ParseMemProfile(80    "parse-mem-profile",81    cl::desc("enable memory profile parsing if it's present in the input data, "82             "on by default unless `--itrace` is set."),83    cl::init(true), cl::cat(AggregatorCategory));84 85static cl::opt<unsigned long long>86FilterPID("pid",87  cl::desc("only use samples from process with specified PID"),88  cl::init(0),89  cl::Optional,90  cl::cat(AggregatorCategory));91 92static cl::opt<bool> ImputeTraceFallthrough(93    "impute-trace-fall-through",94    cl::desc("impute missing fall-throughs for branch-only traces"),95    cl::Optional, cl::cat(AggregatorCategory));96 97static cl::opt<bool>98IgnoreBuildID("ignore-build-id",99  cl::desc("continue even if build-ids in input binary and perf.data mismatch"),100  cl::init(false),101  cl::cat(AggregatorCategory));102 103static cl::opt<bool> IgnoreInterruptLBR(104    "ignore-interrupt-lbr",105    cl::desc("ignore kernel interrupt LBR that happens asynchronously"),106    cl::init(true), cl::cat(AggregatorCategory));107 108static cl::opt<unsigned long long>109MaxSamples("max-samples",110  cl::init(-1ULL),111  cl::desc("maximum number of samples to read from LBR profile"),112  cl::Optional,113  cl::Hidden,114  cl::cat(AggregatorCategory));115 116extern cl::opt<opts::ProfileFormatKind> ProfileFormat;117extern cl::opt<bool> ProfileWritePseudoProbes;118extern cl::opt<std::string> SaveProfile;119 120cl::opt<bool> ReadPreAggregated(121    "pa", cl::desc("skip perf and read data from a pre-aggregated file format"),122    cl::cat(AggregatorCategory));123 124cl::opt<std::string>125    ReadPerfEvents("perf-script-events",126                   cl::desc("skip perf event collection by supplying a "127                            "perf-script output in a textual format"),128                   cl::ReallyHidden, cl::init(""), cl::cat(AggregatorCategory));129 130static cl::opt<bool>131TimeAggregator("time-aggr",132  cl::desc("time BOLT aggregator"),133  cl::init(false),134  cl::ZeroOrMore,135  cl::cat(AggregatorCategory));136 137} // namespace opts138 139namespace {140 141const char TimerGroupName[] = "aggregator";142const char TimerGroupDesc[] = "Aggregator";143 144std::vector<SectionNameAndRange> getTextSections(const BinaryContext *BC) {145  std::vector<SectionNameAndRange> sections;146  for (BinarySection &Section : BC->sections()) {147    if (!Section.isText())148      continue;149    if (Section.getSize() == 0)150      continue;151    sections.push_back(152        {Section.getName(), Section.getAddress(), Section.getEndAddress()});153  }154  llvm::sort(sections,155             [](const SectionNameAndRange &A, const SectionNameAndRange &B) {156               return A.BeginAddress < B.BeginAddress;157             });158  return sections;159}160}161 162DataAggregator::~DataAggregator() { deleteTempFiles(); }163 164namespace {165void deleteTempFile(const std::string &FileName) {166  if (std::error_code Errc = sys::fs::remove(FileName.c_str()))167    errs() << "PERF2BOLT: failed to delete temporary file " << FileName168           << " with error " << Errc.message() << "\n";169}170}171 172void DataAggregator::deleteTempFiles() {173  for (std::string &FileName : TempFiles)174    deleteTempFile(FileName);175  TempFiles.clear();176}177 178void DataAggregator::findPerfExecutable() {179  std::optional<std::string> PerfExecutable =180      sys::Process::FindInEnvPath("PATH", "perf");181  if (!PerfExecutable) {182    outs() << "PERF2BOLT: No perf executable found!\n";183    exit(1);184  }185  PerfPath = *PerfExecutable;186}187 188void DataAggregator::start() {189  outs() << "PERF2BOLT: Starting data aggregation job for " << Filename << "\n";190 191  // Turn on heatmap building if requested by --heatmap flag.192  if (!opts::HeatmapMode && opts::HeatmapOutput.getNumOccurrences())193    opts::HeatmapMode = opts::HeatmapModeKind::HM_Optional;194 195  // Don't launch perf for pre-aggregated files or when perf input is specified196  // by the user.197  if (opts::ReadPreAggregated || !opts::ReadPerfEvents.empty())198    return;199 200  findPerfExecutable();201 202  if (opts::ArmSPE) {203    // pid    from_ip      to_ip        flags204    // where flags could be:205    // P/M: whether branch was Predicted or Mispredicted.206    // N: optionally appears when the branch was Not-Taken (ie fall-through)207    // 12345  0x123/0x456/PN/-/-/8/RET/-208    opts::ITraceAggregation = "bl";209    opts::ParseMemProfile = true;210    opts::BasicAggregation = false;211  }212 213  if (opts::BasicAggregation) {214    launchPerfProcess("events without brstack", MainEventsPPI,215                      "script -F pid,event,ip");216  } else if (!opts::ITraceAggregation.empty()) {217    // Disable parsing memory profile from trace data, unless requested by user.218    if (!opts::ParseMemProfile.getNumOccurrences())219      opts::ParseMemProfile = false;220    launchPerfProcess("branch events with itrace", MainEventsPPI,221                      "script -F pid,brstack --itrace=" +222                          opts::ITraceAggregation);223  } else {224    launchPerfProcess("branch events", MainEventsPPI, "script -F pid,brstack");225  }226 227  if (opts::ParseMemProfile)228    launchPerfProcess("mem events", MemEventsPPI,229                      "script -F pid,event,addr,ip");230 231  launchPerfProcess("process events", MMapEventsPPI,232                    "script --show-mmap-events --no-itrace");233 234  launchPerfProcess("task events", TaskEventsPPI,235                    "script --show-task-events --no-itrace");236}237 238void DataAggregator::abort() {239  if (opts::ReadPreAggregated)240    return;241 242  std::string Error;243 244  // Kill subprocesses in case they are not finished245  sys::Wait(TaskEventsPPI.PI, 1, &Error);246  sys::Wait(MMapEventsPPI.PI, 1, &Error);247  sys::Wait(MainEventsPPI.PI, 1, &Error);248  if (opts::ParseMemProfile)249    sys::Wait(MemEventsPPI.PI, 1, &Error);250 251  deleteTempFiles();252 253  exit(1);254}255 256void DataAggregator::launchPerfProcess(StringRef Name, PerfProcessInfo &PPI,257                                       StringRef Args) {258  SmallVector<StringRef, 4> Argv;259 260  outs() << "PERF2BOLT: spawning perf job to read " << Name << '\n';261  Argv.push_back(PerfPath.data());262 263  Args.split(Argv, ' ');264  Argv.push_back("-f");265  Argv.push_back("-i");266  Argv.push_back(Filename.c_str());267 268  if (std::error_code Errc =269          sys::fs::createTemporaryFile("perf.script", "out", PPI.StdoutPath)) {270    errs() << "PERF2BOLT: failed to create temporary file " << PPI.StdoutPath271           << " with error " << Errc.message() << "\n";272    exit(1);273  }274  TempFiles.push_back(PPI.StdoutPath.data());275 276  if (std::error_code Errc =277          sys::fs::createTemporaryFile("perf.script", "err", PPI.StderrPath)) {278    errs() << "PERF2BOLT: failed to create temporary file " << PPI.StderrPath279           << " with error " << Errc.message() << "\n";280    exit(1);281  }282  TempFiles.push_back(PPI.StderrPath.data());283 284  std::optional<StringRef> Redirects[] = {285      std::nullopt,                      // Stdin286      StringRef(PPI.StdoutPath.data()),  // Stdout287      StringRef(PPI.StderrPath.data())}; // Stderr288 289  LLVM_DEBUG({290    dbgs() << "Launching perf: ";291    for (StringRef Arg : Argv)292      dbgs() << Arg << " ";293    dbgs() << " 1> " << PPI.StdoutPath.data() << " 2> " << PPI.StderrPath.data()294           << "\n";295  });296 297  PPI.PI = sys::ExecuteNoWait(PerfPath.data(), Argv, /*envp*/ std::nullopt,298                              Redirects);299}300 301void DataAggregator::processFileBuildID(StringRef FileBuildID) {302  auto WarningCallback = [](int ReturnCode, StringRef ErrBuf) {303    errs() << "PERF-ERROR: return code " << ReturnCode << "\n" << ErrBuf;304  };305 306  PerfProcessInfo BuildIDProcessInfo;307  launchPerfProcess("buildid list", BuildIDProcessInfo, "buildid-list");308  if (prepareToParse("buildid", BuildIDProcessInfo, WarningCallback))309    return;310 311  std::optional<StringRef> FileName = getFileNameForBuildID(FileBuildID);312  if (FileName && *FileName == sys::path::filename(BC->getFilename())) {313    outs() << "PERF2BOLT: matched build-id and file name\n";314    return;315  }316 317  if (FileName) {318    errs() << "PERF2BOLT-WARNING: build-id matched a different file name\n";319    BuildIDBinaryName = std::string(*FileName);320    return;321  }322 323  if (!hasAllBuildIDs()) {324    errs() << "PERF2BOLT-WARNING: build-id will not be checked because perf "325              "data was recorded without it\n";326    return;327  }328 329  errs() << "PERF2BOLT-ERROR: failed to match build-id from perf output. "330            "This indicates the input binary supplied for data aggregation "331            "is not the same recorded by perf when collecting profiling "332            "data, or there were no samples recorded for the binary. "333            "Use -ignore-build-id option to override.\n";334  if (!opts::IgnoreBuildID)335    abort();336}337 338bool DataAggregator::checkPerfDataMagic(StringRef FileName) {339  if (opts::ReadPreAggregated)340    return true;341 342  Expected<sys::fs::file_t> FD = sys::fs::openNativeFileForRead(FileName);343  if (!FD) {344    consumeError(FD.takeError());345    return false;346  }347 348  char Buf[7] = {0, 0, 0, 0, 0, 0, 0};349 350  auto Close = make_scope_exit([&] { sys::fs::closeFile(*FD); });351  Expected<size_t> BytesRead = sys::fs::readNativeFileSlice(352      *FD, MutableArrayRef(Buf, sizeof(Buf)), 0);353  if (!BytesRead) {354    consumeError(BytesRead.takeError());355    return false;356  }357 358  if (*BytesRead != 7)359    return false;360 361  if (strncmp(Buf, "PERFILE", 7) == 0)362    return true;363  return false;364}365 366void DataAggregator::parsePreAggregated() {367  ErrorOr<std::unique_ptr<MemoryBuffer>> MB =368      MemoryBuffer::getFileOrSTDIN(Filename);369  if (std::error_code EC = MB.getError()) {370    errs() << "PERF2BOLT-ERROR: cannot open " << Filename << ": "371           << EC.message() << "\n";372    exit(1);373  }374 375  FileBuf = std::move(*MB);376  ParsingBuf = FileBuf->getBuffer();377  Col = 0;378  Line = 1;379  if (parsePreAggregatedLBRSamples()) {380    errs() << "PERF2BOLT: failed to parse samples\n";381    exit(1);382  }383}384 385void DataAggregator::filterBinaryMMapInfo() {386  if (opts::FilterPID) {387    auto MMapInfoIter = BinaryMMapInfo.find(opts::FilterPID);388    if (MMapInfoIter != BinaryMMapInfo.end()) {389      MMapInfo MMap = MMapInfoIter->second;390      BinaryMMapInfo.clear();391      BinaryMMapInfo.insert(std::make_pair(MMap.PID, MMap));392    } else {393      if (errs().has_colors())394        errs().changeColor(raw_ostream::RED);395      errs() << "PERF2BOLT-ERROR: could not find a profile matching PID \""396             << opts::FilterPID << "\""397             << " for binary \"" << BC->getFilename() << "\".";398      assert(!BinaryMMapInfo.empty() && "No memory map for matching binary");399      errs() << " Profile for the following process is available:\n";400      for (std::pair<const uint64_t, MMapInfo> &MMI : BinaryMMapInfo)401        outs() << "  " << MMI.second.PID402               << (MMI.second.Forked ? " (forked)\n" : "\n");403 404      if (errs().has_colors())405        errs().resetColor();406 407      exit(1);408    }409  }410}411 412int DataAggregator::prepareToParse(StringRef Name, PerfProcessInfo &Process,413                                   PerfProcessErrorCallbackTy Callback) {414  if (!opts::ReadPerfEvents.empty()) {415    outs() << "PERF2BOLT: using pre-processed perf events for '" << Name416           << "' (perf-script-events)\n";417    ParsingBuf = opts::ReadPerfEvents;418    return 0;419  }420 421  std::string Error;422  outs() << "PERF2BOLT: waiting for perf " << Name423         << " collection to finish...\n";424  std::optional<sys::ProcessStatistics> PS;425  sys::ProcessInfo PI = sys::Wait(Process.PI, std::nullopt, &Error, &PS);426 427  if (!Error.empty()) {428    errs() << "PERF-ERROR: " << PerfPath << ": " << Error << "\n";429    deleteTempFiles();430    exit(1);431  }432 433  LLVM_DEBUG({434    const float UserSec = 1.f * PS->UserTime.count() / 1e6;435    const float TotalSec = 1.f * PS->TotalTime.count() / 1e6;436    const float PeakGiB = 1.f * PS->PeakMemory / (1 << 20);437    dbgs() << formatv("Finished in {0:f2}s user time, {1:f2}s total time, "438                      "{2:f2} GiB peak RSS\n",439                      UserSec, TotalSec, PeakGiB);440  });441 442  if (PI.ReturnCode != 0) {443    ErrorOr<std::unique_ptr<MemoryBuffer>> ErrorMB =444        MemoryBuffer::getFileOrSTDIN(Process.StderrPath.data());445    StringRef ErrBuf = (*ErrorMB)->getBuffer();446 447    deleteTempFiles();448    Callback(PI.ReturnCode, ErrBuf);449    return PI.ReturnCode;450  }451 452  ErrorOr<std::unique_ptr<MemoryBuffer>> MB =453      MemoryBuffer::getFileOrSTDIN(Process.StdoutPath.data());454  if (std::error_code EC = MB.getError()) {455    errs() << "Cannot open " << Process.StdoutPath.data() << ": "456           << EC.message() << "\n";457    deleteTempFiles();458    exit(1);459  }460 461  FileBuf = std::move(*MB);462  ParsingBuf = FileBuf->getBuffer();463  Col = 0;464  Line = 1;465  return PI.ReturnCode;466}467 468void DataAggregator::parsePerfData(BinaryContext &BC) {469  auto ErrorCallback = [](int ReturnCode, StringRef ErrBuf) {470    errs() << "PERF-ERROR: return code " << ReturnCode << "\n" << ErrBuf;471    exit(1);472  };473 474  auto MemEventsErrorCallback = [&](int ReturnCode, StringRef ErrBuf) {475    Regex NoData("Samples for '.*' event do not have ADDR attribute set. "476                 "Cannot print 'addr' field.");477    if (!NoData.match(ErrBuf))478      ErrorCallback(ReturnCode, ErrBuf);479  };480 481  if (std::optional<StringRef> FileBuildID = BC.getFileBuildID()) {482    outs() << "BOLT-INFO: binary build-id is:     " << *FileBuildID << "\n";483    processFileBuildID(*FileBuildID);484  } else {485    errs() << "BOLT-WARNING: build-id will not be checked because we could "486              "not read one from input binary\n";487  }488 489  if (BC.IsLinuxKernel) {490    // Current MMap parsing logic does not work with linux kernel.491    // MMap entries for linux kernel uses PERF_RECORD_MMAP492    // format instead of typical PERF_RECORD_MMAP2 format.493    // Since linux kernel address mapping is absolute (same as494    // in the ELF file), we avoid parsing MMap in linux kernel mode.495    // While generating optimized linux kernel binary, we may need496    // to parse MMap entries.497 498    // In linux kernel mode, we analyze and optimize499    // all linux kernel binary instructions, irrespective500    // of whether they are due to system calls or due to501    // interrupts. Therefore, we cannot ignore interrupt502    // in Linux kernel mode.503    opts::IgnoreInterruptLBR = false;504  } else {505    prepareToParse("mmap events", MMapEventsPPI, ErrorCallback);506    if (parseMMapEvents())507      errs() << "PERF2BOLT: failed to parse mmap events\n";508  }509 510  prepareToParse("task events", TaskEventsPPI, ErrorCallback);511  if (parseTaskEvents())512    errs() << "PERF2BOLT: failed to parse task events\n";513 514  filterBinaryMMapInfo();515  prepareToParse("events", MainEventsPPI, ErrorCallback);516 517  if ((!opts::BasicAggregation && parseBranchEvents()) ||518      (opts::BasicAggregation && parseBasicEvents()))519    errs() << "PERF2BOLT: failed to parse samples\n";520 521  // Special handling for memory events522  if (opts::ParseMemProfile &&523      !prepareToParse("mem events", MemEventsPPI, MemEventsErrorCallback))524    if (const std::error_code EC = parseMemEvents())525      errs() << "PERF2BOLT: failed to parse memory events: " << EC.message()526             << '\n';527 528  deleteTempFiles();529}530 531void DataAggregator::imputeFallThroughs() {532  if (Traces.empty())533    return;534 535  std::pair PrevBranch(Trace::EXTERNAL, Trace::EXTERNAL);536  uint64_t AggregateCount = 0;537  uint64_t AggregateFallthroughSize = 0;538  uint64_t InferredTraces = 0;539 540  // Helper map with whether the instruction is a call/ret/unconditional branch541  std::unordered_map<uint64_t, bool> IsUncondCTMap;542  auto checkUnconditionalControlTransfer = [&](const uint64_t Addr) {543    auto isUncondCT = [&](const MCInst &MI) -> bool {544      return BC->MIB->isUnconditionalControlTransfer(MI);545    };546    return testAndSet<bool>(Addr, isUncondCT, IsUncondCTMap).value_or(true);547  };548 549  // Traces are sorted by their component addresses (Branch, From, To).550  // assert(is_sorted(Traces));551 552  // Traces corresponding to the top-of-stack branch entry with a missing553  // fall-through have BR_ONLY(-1ULL/UINT64_MAX) in To field, meaning that for554  // fixed values of Branch and From branch-only traces are stored after all555  // traces with valid fall-through.556  //557  // Group traces by (Branch, From) and compute weighted average fall-through558  // length for the top-of-stack trace (closing the group) by accumulating the559  // fall-through lengths of traces with valid fall-throughs earlier in the560  // group.561  for (auto &[Trace, Info] : Traces) {562    // Skip fall-throughs in external code.563    if (Trace.From == Trace::EXTERNAL)564      continue;565    if (std::pair CurrentBranch(Trace.Branch, Trace.From);566        CurrentBranch != PrevBranch) {567      // New group: reset aggregates.568      AggregateCount = AggregateFallthroughSize = 0;569      PrevBranch = CurrentBranch;570    }571    // BR_ONLY must be the last trace in the group572    if (Trace.To == Trace::BR_ONLY) {573      // If the group is not empty, use aggregate values, otherwise 0-length574      // for unconditional jumps (call/ret/uncond branch) or 1-length for others575      uint64_t InferredBytes =576          AggregateFallthroughSize577              ? AggregateFallthroughSize / AggregateCount578              : !checkUnconditionalControlTransfer(Trace.From);579      Trace.To = Trace.From + InferredBytes;580      LLVM_DEBUG(dbgs() << "imputed " << Trace << " (" << InferredBytes581                        << " bytes)\n");582      ++InferredTraces;583    } else {584      // Only use valid fall-through lengths585      if (Trace.To != Trace::EXTERNAL)586        AggregateFallthroughSize += (Trace.To - Trace.From) * Info.TakenCount;587      AggregateCount += Info.TakenCount;588    }589  }590  if (opts::Verbosity >= 1)591    outs() << "BOLT-INFO: imputed " << InferredTraces << " traces\n";592}593 594Error DataAggregator::preprocessProfile(BinaryContext &BC) {595  this->BC = &BC;596 597  if (opts::ReadPreAggregated) {598    parsePreAggregated();599  } else {600    parsePerfData(BC);601  }602 603  // Sort parsed traces for faster processing.604  llvm::sort(Traces, llvm::less_first());605 606  if (opts::ImputeTraceFallthrough)607    imputeFallThroughs();608 609  if (opts::HeatmapMode) {610    if (std::error_code EC = printLBRHeatMap())611      return errorCodeToError(EC);612    if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive)613      exit(0);614  }615 616  return Error::success();617}618 619Error DataAggregator::readProfile(BinaryContext &BC) {620  processProfile(BC);621 622  for (auto &BFI : BC.getBinaryFunctions()) {623    BinaryFunction &Function = BFI.second;624    convertBranchData(Function);625  }626 627  if (opts::AggregateOnly) {628    if (opts::ProfileFormat == opts::ProfileFormatKind::PF_Fdata)629      if (std::error_code EC = writeAggregatedFile(opts::OutputFilename))630        report_error("cannot create output data file", EC);631 632    // BAT YAML is handled by DataAggregator since normal YAML output requires633    // CFG which is not available in BAT mode.634    if (usesBAT()) {635      if (opts::ProfileFormat == opts::ProfileFormatKind::PF_YAML)636        if (std::error_code EC = writeBATYAML(BC, opts::OutputFilename))637          report_error("cannot create output data file", EC);638      if (!opts::SaveProfile.empty())639        if (std::error_code EC = writeBATYAML(BC, opts::SaveProfile))640          report_error("cannot create output data file", EC);641    }642  }643 644  return Error::success();645}646 647bool DataAggregator::mayHaveProfileData(const BinaryFunction &Function) {648  return Function.hasProfileAvailable();649}650 651void DataAggregator::processProfile(BinaryContext &BC) {652  if (opts::BasicAggregation)653    processBasicEvents();654  else655    processBranchEvents();656 657  processMemEvents();658 659  // Mark all functions with registered events as having a valid profile.660  for (auto &BFI : BC.getBinaryFunctions()) {661    BinaryFunction &BF = BFI.second;662    if (FuncBranchData *FBD = getBranchData(BF)) {663      BF.markProfiled(BinaryFunction::PF_BRANCH);664      BF.RawSampleCount = FBD->getNumExecutedBranches();665    } else if (FuncBasicSampleData *FSD =666                   getFuncBasicSampleData(BF.getNames())) {667      BF.markProfiled(BinaryFunction::PF_BASIC);668      BF.RawSampleCount = FSD->getSamples();669    }670  }671 672  for (auto &FuncBranches : NamesToBranches) {673    llvm::stable_sort(FuncBranches.second.Data);674    llvm::stable_sort(FuncBranches.second.EntryData);675  }676 677  for (auto &MemEvents : NamesToMemEvents)678    llvm::stable_sort(MemEvents.second.Data);679 680  // Release intermediate storage.681  clear(Traces);682  clear(BasicSamples);683  clear(MemSamples);684}685 686BinaryFunction *687DataAggregator::getBinaryFunctionContainingAddress(uint64_t Address) const {688  if (!BC->containsAddress(Address))689    return nullptr;690 691  return BC->getBinaryFunctionContainingAddress(Address, /*CheckPastEnd=*/false,692                                                /*UseMaxSize=*/true);693}694 695BinaryFunction *696DataAggregator::getBATParentFunction(const BinaryFunction &Func) const {697  if (BAT)698    if (const uint64_t HotAddr = BAT->fetchParentAddress(Func.getAddress()))699      return getBinaryFunctionContainingAddress(HotAddr);700  return nullptr;701}702 703StringRef DataAggregator::getLocationName(const BinaryFunction &Func,704                                          bool BAT) {705  if (!BAT)706    return Func.getOneName();707 708  const BinaryFunction *OrigFunc = &Func;709  // If it is a local function, prefer the name containing the file name where710  // the local function was declared711  for (StringRef AlternativeName : OrigFunc->getNames()) {712    size_t FileNameIdx = AlternativeName.find('/');713    // Confirm the alternative name has the pattern Symbol/FileName/1 before714    // using it715    if (FileNameIdx == StringRef::npos ||716        AlternativeName.find('/', FileNameIdx + 1) == StringRef::npos)717      continue;718    return AlternativeName;719  }720  return OrigFunc->getOneName();721}722 723bool DataAggregator::doBasicSample(BinaryFunction &OrigFunc, uint64_t Address,724                                   uint64_t Count) {725  // To record executed bytes, use basic block size as is regardless of BAT.726  uint64_t BlockSize = 0;727  if (BinaryBasicBlock *BB = OrigFunc.getBasicBlockContainingOffset(728          Address - OrigFunc.getAddress()))729    BlockSize = BB->getOriginalSize();730 731  BinaryFunction *ParentFunc = getBATParentFunction(OrigFunc);732  BinaryFunction &Func = ParentFunc ? *ParentFunc : OrigFunc;733  // Attach executed bytes to parent function in case of cold fragment.734  Func.SampleCountInBytes += Count * BlockSize;735 736  auto I = NamesToBasicSamples.find(Func.getOneName());737  if (I == NamesToBasicSamples.end()) {738    bool Success;739    StringRef LocName = getLocationName(Func, BAT);740    std::tie(I, Success) = NamesToBasicSamples.insert(std::make_pair(741        Func.getOneName(),742        FuncBasicSampleData(LocName, FuncBasicSampleData::ContainerTy())));743  }744 745  Address -= Func.getAddress();746  if (BAT)747    Address = BAT->translate(Func.getAddress(), Address, /*IsBranchSrc=*/false);748 749  I->second.bumpCount(Address, Count);750  return true;751}752 753bool DataAggregator::doIntraBranch(BinaryFunction &Func, uint64_t From,754                                   uint64_t To, uint64_t Count,755                                   uint64_t Mispreds) {756  FuncBranchData *AggrData = getBranchData(Func);757  if (!AggrData) {758    AggrData = &NamesToBranches[Func.getOneName()];759    AggrData->Name = getLocationName(Func, BAT);760    setBranchData(Func, AggrData);761  }762 763  LLVM_DEBUG(dbgs() << "BOLT-DEBUG: bumpBranchCount: "764                    << formatv("{0} @ {1:x} -> {0} @ {2:x}\n", Func, From, To));765  AggrData->bumpBranchCount(From, To, Count, Mispreds);766  return true;767}768 769bool DataAggregator::doInterBranch(BinaryFunction *FromFunc,770                                   BinaryFunction *ToFunc, uint64_t From,771                                   uint64_t To, uint64_t Count,772                                   uint64_t Mispreds) {773  FuncBranchData *FromAggrData = nullptr;774  FuncBranchData *ToAggrData = nullptr;775  StringRef SrcFunc;776  StringRef DstFunc;777  if (FromFunc) {778    SrcFunc = getLocationName(*FromFunc, BAT);779    FromAggrData = getBranchData(*FromFunc);780    if (!FromAggrData) {781      FromAggrData = &NamesToBranches[FromFunc->getOneName()];782      FromAggrData->Name = SrcFunc;783      setBranchData(*FromFunc, FromAggrData);784    }785 786    recordExit(*FromFunc, From, Mispreds, Count);787  }788  if (ToFunc) {789    DstFunc = getLocationName(*ToFunc, BAT);790    ToAggrData = getBranchData(*ToFunc);791    if (!ToAggrData) {792      ToAggrData = &NamesToBranches[ToFunc->getOneName()];793      ToAggrData->Name = DstFunc;794      setBranchData(*ToFunc, ToAggrData);795    }796 797    recordEntry(*ToFunc, To, Mispreds, Count);798  }799 800  if (FromAggrData)801    FromAggrData->bumpCallCount(From, Location(!DstFunc.empty(), DstFunc, To),802                                Count, Mispreds);803  if (ToAggrData)804    ToAggrData->bumpEntryCount(Location(!SrcFunc.empty(), SrcFunc, From), To,805                               Count, Mispreds);806  return true;807}808 809bool DataAggregator::checkReturn(uint64_t Addr) {810  auto isReturn = [&](const MCInst &MI) -> bool {811    return BC->MIB->isReturn(MI);812  };813  return testAndSet<bool>(Addr, isReturn, Returns).value_or(false);814}815 816bool DataAggregator::doBranch(uint64_t From, uint64_t To, uint64_t Count,817                              uint64_t Mispreds) {818  // Mutates \p Addr to an offset into the containing function, performing BAT819  // offset translation and parent lookup.820  //821  // Returns the containing function (or BAT parent).822  auto handleAddress = [&](uint64_t &Addr, bool IsFrom) {823    BinaryFunction *Func = getBinaryFunctionContainingAddress(Addr);824    if (!Func) {825      Addr = 0;826      return Func;827    }828 829    Addr -= Func->getAddress();830 831    if (BAT)832      Addr = BAT->translate(Func->getAddress(), Addr, IsFrom);833 834    if (BinaryFunction *ParentFunc = getBATParentFunction(*Func))835      return ParentFunc;836 837    return Func;838  };839 840  BinaryFunction *FromFunc = handleAddress(From, /*IsFrom*/ true);841  BinaryFunction *ToFunc = handleAddress(To, /*IsFrom*/ false);842  if (!FromFunc && !ToFunc)843    return false;844 845  // Treat recursive control transfers as inter-branches.846  if (FromFunc == ToFunc && To != 0) {847    recordBranch(*FromFunc, From, To, Count, Mispreds);848    return doIntraBranch(*FromFunc, From, To, Count, Mispreds);849  }850 851  return doInterBranch(FromFunc, ToFunc, From, To, Count, Mispreds);852}853 854bool DataAggregator::doTrace(const Trace &Trace, uint64_t Count,855                             bool IsReturn) {856  const uint64_t From = Trace.From, To = Trace.To;857  BinaryFunction *FromFunc = getBinaryFunctionContainingAddress(From);858  BinaryFunction *ToFunc = getBinaryFunctionContainingAddress(To);859  NumTraces += Count;860  if (!FromFunc || !ToFunc) {861    LLVM_DEBUG(dbgs() << "Out of range trace " << Trace << '\n');862    NumLongRangeTraces += Count;863    return false;864  }865  if (FromFunc != ToFunc) {866    LLVM_DEBUG(dbgs() << "Invalid trace " << Trace << '\n');867    NumInvalidTraces += Count;868    return false;869  }870 871  // Set ParentFunc to BAT parent function or FromFunc itself.872  BinaryFunction *ParentFunc = getBATParentFunction(*FromFunc);873  if (!ParentFunc)874    ParentFunc = FromFunc;875  ParentFunc->SampleCountInBytes += Count * (To - From);876 877  const uint64_t FuncAddress = FromFunc->getAddress();878  std::optional<BoltAddressTranslation::FallthroughListTy> FTs =879      BAT && BAT->isBATFunction(FuncAddress)880          ? BAT->getFallthroughsInTrace(FuncAddress, From - IsReturn, To)881          : getFallthroughsInTrace(*FromFunc, Trace, Count, IsReturn);882  if (!FTs) {883    LLVM_DEBUG(dbgs() << "Invalid trace " << Trace << '\n');884    NumInvalidTraces += Count;885    return false;886  }887 888  LLVM_DEBUG(dbgs() << "Processing " << FTs->size() << " fallthroughs for "889                    << FromFunc->getPrintName() << ":" << Trace << '\n');890  for (const auto &[From, To] : *FTs)891    doIntraBranch(*ParentFunc, From, To, Count, false);892 893  return true;894}895 896std::optional<SmallVector<std::pair<uint64_t, uint64_t>, 16>>897DataAggregator::getFallthroughsInTrace(BinaryFunction &BF, const Trace &Trace,898                                       uint64_t Count, bool IsReturn) const {899  SmallVector<std::pair<uint64_t, uint64_t>, 16> Branches;900 901  BinaryContext &BC = BF.getBinaryContext();902 903  // Offsets of the trace within this function.904  const uint64_t From = Trace.From - BF.getAddress();905  const uint64_t To = Trace.To - BF.getAddress();906 907  if (From > To)908    return std::nullopt;909 910  // Accept fall-throughs inside pseudo functions (PLT/thunks).911  // This check has to be above BF.empty as pseudo functions would pass it:912  // pseudo => ignored => CFG not built => empty.913  // If we return nullopt, trace would be reported as mismatching disassembled914  // function contents which it is not. To avoid this, return an empty915  // fall-through list instead.916  if (BF.isPseudo())917    return Branches;918 919  // Can only record traces in CFG state920  if (!BF.hasCFG())921    return std::nullopt;922 923  const BinaryBasicBlock *FromBB = BF.getBasicBlockContainingOffset(From);924  const BinaryBasicBlock *ToBB = BF.getBasicBlockContainingOffset(To);925 926  if (!FromBB || !ToBB)927    return std::nullopt;928 929  // Adjust FromBB if the first LBR is a return from the last instruction in930  // the previous block (that instruction should be a call).931  if (Trace.Branch != Trace::FT_ONLY && !BF.containsAddress(Trace.Branch) &&932      From == FromBB->getOffset() &&933      (IsReturn ? From : !(FromBB->isEntryPoint() || FromBB->isLandingPad()))) {934    const BinaryBasicBlock *PrevBB =935        BF.getLayout().getBlock(FromBB->getIndex() - 1);936    if (PrevBB->getSuccessor(FromBB->getLabel())) {937      const MCInst *Instr = PrevBB->getLastNonPseudoInstr();938      if (Instr && BC.MIB->isCall(*Instr))939        FromBB = PrevBB;940      else941        LLVM_DEBUG(dbgs() << "invalid trace (no call): " << Trace << '\n');942    } else {943      LLVM_DEBUG(dbgs() << "invalid trace: " << Trace << '\n');944    }945  }946 947  // Fill out information for fall-through edges. The From and To could be948  // within the same basic block, e.g. when two call instructions are in the949  // same block. In this case we skip the processing.950  if (FromBB == ToBB)951    return Branches;952 953  // Process blocks in the original layout order.954  BinaryBasicBlock *BB = BF.getLayout().getBlock(FromBB->getIndex());955  assert(BB == FromBB && "index mismatch");956  while (BB != ToBB) {957    BinaryBasicBlock *NextBB = BF.getLayout().getBlock(BB->getIndex() + 1);958    assert((NextBB && NextBB->getOffset() > BB->getOffset()) && "bad layout");959 960    // Check for bad LBRs.961    if (!BB->getSuccessor(NextBB->getLabel())) {962      LLVM_DEBUG(dbgs() << "no fall-through for the trace: " << Trace << '\n');963      return std::nullopt;964    }965 966    const MCInst *Instr = BB->getLastNonPseudoInstr();967    uint64_t Offset = 0;968    if (Instr)969      Offset = BC.MIB->getOffsetWithDefault(*Instr, 0);970    else971      Offset = BB->getOffset();972 973    Branches.emplace_back(Offset, NextBB->getOffset());974 975    BB = NextBB;976  }977 978  // Record fall-through jumps979  for (const auto &[FromOffset, ToOffset] : Branches) {980    BinaryBasicBlock *FromBB = BF.getBasicBlockContainingOffset(FromOffset);981    BinaryBasicBlock *ToBB = BF.getBasicBlockAtOffset(ToOffset);982    assert(FromBB && ToBB);983    BinaryBasicBlock::BinaryBranchInfo &BI = FromBB->getBranchInfo(*ToBB);984    BI.Count += Count;985  }986 987  return Branches;988}989 990bool DataAggregator::recordEntry(BinaryFunction &BF, uint64_t To, bool Mispred,991                                 uint64_t Count) const {992  if (To > BF.getSize())993    return false;994 995  if (!BF.hasProfile())996    BF.ExecutionCount = 0;997 998  BinaryBasicBlock *EntryBB = nullptr;999  if (To == 0) {1000    BF.ExecutionCount += Count;1001    if (!BF.empty())1002      EntryBB = &BF.front();1003  } else if (BinaryBasicBlock *BB = BF.getBasicBlockAtOffset(To)) {1004    if (BB->isEntryPoint())1005      EntryBB = BB;1006  }1007 1008  if (EntryBB)1009    EntryBB->setExecutionCount(EntryBB->getKnownExecutionCount() + Count);1010 1011  return true;1012}1013 1014bool DataAggregator::recordExit(BinaryFunction &BF, uint64_t From, bool Mispred,1015                                uint64_t Count) const {1016  if (!BF.isSimple() || From > BF.getSize())1017    return false;1018 1019  if (!BF.hasProfile())1020    BF.ExecutionCount = 0;1021 1022  return true;1023}1024 1025ErrorOr<DataAggregator::LBREntry> DataAggregator::parseLBREntry() {1026  LBREntry Res;1027  ErrorOr<StringRef> FromStrRes = parseString('/');1028  if (std::error_code EC = FromStrRes.getError())1029    return EC;1030  StringRef OffsetStr = FromStrRes.get();1031  if (OffsetStr.getAsInteger(0, Res.From)) {1032    reportError("expected hexadecimal number with From address");1033    Diag << "Found: " << OffsetStr << "\n";1034    return make_error_code(llvm::errc::io_error);1035  }1036 1037  ErrorOr<StringRef> ToStrRes = parseString('/');1038  if (std::error_code EC = ToStrRes.getError())1039    return EC;1040  OffsetStr = ToStrRes.get();1041  if (OffsetStr.getAsInteger(0, Res.To)) {1042    reportError("expected hexadecimal number with To address");1043    Diag << "Found: " << OffsetStr << "\n";1044    return make_error_code(llvm::errc::io_error);1045  }1046 1047  ErrorOr<StringRef> MispredStrRes = parseString('/');1048  if (std::error_code EC = MispredStrRes.getError())1049    return EC;1050  StringRef MispredStr = MispredStrRes.get();1051  // SPE brstack mispredicted flags might be up to two characters long:1052  // 'PN' or 'MN'. Where 'N' optionally appears.1053  bool ValidStrSize = opts::ArmSPE1054                          ? MispredStr.size() >= 1 && MispredStr.size() <= 21055                          : MispredStr.size() == 1;1056  bool SpeTakenBitErr =1057      (opts::ArmSPE && MispredStr.size() == 2 && MispredStr[1] != 'N');1058  bool PredictionBitErr =1059      !ValidStrSize ||1060      (MispredStr[0] != 'P' && MispredStr[0] != 'M' && MispredStr[0] != '-');1061  if (SpeTakenBitErr)1062    reportError("expected 'N' as SPE prediction bit for a not-taken branch");1063  if (PredictionBitErr)1064    reportError("expected 'P', 'M' or '-' char as a prediction bit");1065 1066  if (SpeTakenBitErr || PredictionBitErr) {1067    Diag << "Found: " << MispredStr << "\n";1068    return make_error_code(llvm::errc::io_error);1069  }1070  Res.Mispred = MispredStr[0] == 'M';1071 1072  static bool MispredWarning = true;1073  if (MispredStr[0] == '-' && MispredWarning) {1074    errs() << "PERF2BOLT-WARNING: misprediction bit is missing in profile\n";1075    MispredWarning = false;1076  }1077 1078  ErrorOr<StringRef> Rest = parseString(FieldSeparator, true);1079  if (std::error_code EC = Rest.getError())1080    return EC;1081  if (Rest.get().size() < 5) {1082    reportError("expected rest of brstack entry");1083    Diag << "Found: " << Rest.get() << "\n";1084    return make_error_code(llvm::errc::io_error);1085  }1086  return Res;1087}1088 1089bool DataAggregator::checkAndConsumeFS() {1090  if (ParsingBuf[0] != FieldSeparator)1091    return false;1092 1093  ParsingBuf = ParsingBuf.drop_front(1);1094  Col += 1;1095  return true;1096}1097 1098void DataAggregator::consumeRestOfLine() {1099  size_t LineEnd = ParsingBuf.find_first_of('\n');1100  if (LineEnd == StringRef::npos) {1101    ParsingBuf = StringRef();1102    Col = 0;1103    Line += 1;1104    return;1105  }1106  ParsingBuf = ParsingBuf.drop_front(LineEnd + 1);1107  Col = 0;1108  Line += 1;1109}1110 1111bool DataAggregator::checkNewLine() {1112  return ParsingBuf[0] == '\n';1113}1114 1115ErrorOr<DataAggregator::PerfBranchSample> DataAggregator::parseBranchSample() {1116  PerfBranchSample Res;1117 1118  while (checkAndConsumeFS()) {1119  }1120 1121  ErrorOr<int64_t> PIDRes = parseNumberField(FieldSeparator, true);1122  if (std::error_code EC = PIDRes.getError())1123    return EC;1124  auto MMapInfoIter = BinaryMMapInfo.find(*PIDRes);1125  if (!BC->IsLinuxKernel && MMapInfoIter == BinaryMMapInfo.end()) {1126    consumeRestOfLine();1127    return make_error_code(errc::no_such_process);1128  }1129 1130  if (checkAndConsumeNewLine())1131    return Res;1132 1133  while (!checkAndConsumeNewLine()) {1134    checkAndConsumeFS();1135 1136    ErrorOr<LBREntry> LBRRes = parseLBREntry();1137    if (std::error_code EC = LBRRes.getError())1138      return EC;1139    LBREntry LBR = LBRRes.get();1140    if (ignoreKernelInterrupt(LBR))1141      continue;1142    if (!BC->HasFixedLoadAddress)1143      adjustLBR(LBR, MMapInfoIter->second);1144    Res.LBR.push_back(LBR);1145  }1146 1147  return Res;1148}1149 1150ErrorOr<DataAggregator::PerfBasicSample> DataAggregator::parseBasicSample() {1151  while (checkAndConsumeFS()) {1152  }1153 1154  ErrorOr<int64_t> PIDRes = parseNumberField(FieldSeparator, true);1155  if (std::error_code EC = PIDRes.getError())1156    return EC;1157 1158  auto MMapInfoIter = BinaryMMapInfo.find(*PIDRes);1159  if (MMapInfoIter == BinaryMMapInfo.end()) {1160    consumeRestOfLine();1161    return PerfBasicSample{StringRef(), 0};1162  }1163 1164  while (checkAndConsumeFS()) {1165  }1166 1167  ErrorOr<StringRef> Event = parseString(FieldSeparator);1168  if (std::error_code EC = Event.getError())1169    return EC;1170 1171  while (checkAndConsumeFS()) {1172  }1173 1174  ErrorOr<uint64_t> AddrRes = parseHexField(FieldSeparator, true);1175  if (std::error_code EC = AddrRes.getError())1176    return EC;1177 1178  if (!checkAndConsumeNewLine()) {1179    reportError("expected end of line");1180    return make_error_code(llvm::errc::io_error);1181  }1182 1183  uint64_t Address = *AddrRes;1184  if (!BC->HasFixedLoadAddress)1185    adjustAddress(Address, MMapInfoIter->second);1186 1187  return PerfBasicSample{Event.get(), Address};1188}1189 1190ErrorOr<DataAggregator::PerfMemSample> DataAggregator::parseMemSample() {1191  PerfMemSample Res{0, 0};1192 1193  while (checkAndConsumeFS()) {1194  }1195 1196  ErrorOr<int64_t> PIDRes = parseNumberField(FieldSeparator, true);1197  if (std::error_code EC = PIDRes.getError())1198    return EC;1199 1200  auto MMapInfoIter = BinaryMMapInfo.find(*PIDRes);1201  if (MMapInfoIter == BinaryMMapInfo.end()) {1202    consumeRestOfLine();1203    return Res;1204  }1205 1206  while (checkAndConsumeFS()) {1207  }1208 1209  ErrorOr<StringRef> Event = parseString(FieldSeparator);1210  if (std::error_code EC = Event.getError())1211    return EC;1212  if (!Event.get().contains("mem-loads")) {1213    consumeRestOfLine();1214    return Res;1215  }1216 1217  while (checkAndConsumeFS()) {1218  }1219 1220  ErrorOr<uint64_t> AddrRes = parseHexField(FieldSeparator);1221  if (std::error_code EC = AddrRes.getError())1222    return EC;1223 1224  while (checkAndConsumeFS()) {1225  }1226 1227  ErrorOr<uint64_t> PCRes = parseHexField(FieldSeparator, true);1228  if (std::error_code EC = PCRes.getError()) {1229    consumeRestOfLine();1230    return EC;1231  }1232 1233  if (!checkAndConsumeNewLine()) {1234    reportError("expected end of line");1235    return make_error_code(llvm::errc::io_error);1236  }1237 1238  uint64_t Address = *AddrRes;1239  if (!BC->HasFixedLoadAddress)1240    adjustAddress(Address, MMapInfoIter->second);1241 1242  return PerfMemSample{PCRes.get(), Address};1243}1244 1245ErrorOr<Location> DataAggregator::parseLocationOrOffset() {1246  auto parseOffset = [this]() -> ErrorOr<Location> {1247    ErrorOr<uint64_t> Res = parseHexField(FieldSeparator);1248    if (std::error_code EC = Res.getError())1249      return EC;1250    return Location(Res.get());1251  };1252 1253  size_t Sep = ParsingBuf.find_first_of(" \n");1254  if (Sep == StringRef::npos)1255    return parseOffset();1256  StringRef LookAhead = ParsingBuf.substr(0, Sep);1257  if (!LookAhead.contains(':'))1258    return parseOffset();1259 1260  ErrorOr<StringRef> BuildID = parseString(':');1261  if (std::error_code EC = BuildID.getError())1262    return EC;1263  ErrorOr<uint64_t> Offset = parseHexField(FieldSeparator);1264  if (std::error_code EC = Offset.getError())1265    return EC;1266  return Location(true, BuildID.get(), Offset.get());1267}1268 1269std::error_code DataAggregator::parseAggregatedLBREntry() {1270  enum AggregatedLBREntry : char {1271    INVALID = 0,1272    EVENT_NAME,         // E1273    TRACE,              // T1274    RETURN,             // R1275    SAMPLE,             // S1276    BRANCH,             // B1277    FT,                 // F1278    FT_EXTERNAL_ORIGIN, // f1279    FT_EXTERNAL_RETURN  // r1280  } Type = INVALID;1281 1282  /// The number of fields to parse, set based on \p Type.1283  int AddrNum = 0;1284  int CounterNum = 0;1285  /// Storage for parsed fields.1286  StringRef EventName;1287  std::optional<Location> Addr[3];1288  int64_t Counters[2] = {0};1289 1290  /// Parse strings: record type and optionally an event name.1291  while (Type == INVALID || Type == EVENT_NAME) {1292    while (checkAndConsumeFS()) {1293    }1294    ErrorOr<StringRef> StrOrErr =1295        parseString(FieldSeparator, Type == EVENT_NAME);1296    if (std::error_code EC = StrOrErr.getError())1297      return EC;1298    StringRef Str = StrOrErr.get();1299 1300    if (Type == EVENT_NAME) {1301      EventName = Str;1302      break;1303    }1304 1305    Type = StringSwitch<AggregatedLBREntry>(Str)1306               .Case("T", TRACE)1307               .Case("R", RETURN)1308               .Case("S", SAMPLE)1309               .Case("E", EVENT_NAME)1310               .Case("B", BRANCH)1311               .Case("F", FT)1312               .Case("f", FT_EXTERNAL_ORIGIN)1313               .Case("r", FT_EXTERNAL_RETURN)1314               .Default(INVALID);1315 1316    if (Type == INVALID) {1317      reportError("expected T, R, S, E, B, F, f or r");1318      return make_error_code(llvm::errc::io_error);1319    }1320 1321    using SSI = StringSwitch<int>;1322    AddrNum =1323        SSI(Str).Cases({"T", "R"}, 3).Case("S", 1).Case("E", 0).Default(2);1324    CounterNum = SSI(Str).Case("B", 2).Case("E", 0).Default(1);1325  }1326 1327  /// Parse locations depending on entry type, recording them in \p Addr array.1328  for (int I = 0; I < AddrNum; ++I) {1329    while (checkAndConsumeFS()) {1330    }1331    ErrorOr<Location> AddrOrErr = parseLocationOrOffset();1332    if (std::error_code EC = AddrOrErr.getError())1333      return EC;1334    Addr[I] = AddrOrErr.get();1335  }1336 1337  /// Parse counters depending on entry type.1338  for (int I = 0; I < CounterNum; ++I) {1339    while (checkAndConsumeFS()) {1340    }1341    ErrorOr<int64_t> CountOrErr =1342        parseNumberField(FieldSeparator, I + 1 == CounterNum);1343    if (std::error_code EC = CountOrErr.getError())1344      return EC;1345    Counters[I] = CountOrErr.get();1346  }1347 1348  /// Expect end of line here.1349  if (!checkAndConsumeNewLine()) {1350    reportError("expected end of line");1351    return make_error_code(llvm::errc::io_error);1352  }1353 1354  /// Record event name into \p EventNames and return.1355  if (Type == EVENT_NAME) {1356    EventNames.insert(EventName);1357    return std::error_code();1358  }1359 1360  const uint64_t FromOffset = Addr[0]->Offset;1361  BinaryFunction *FromFunc = getBinaryFunctionContainingAddress(FromOffset);1362  if (FromFunc)1363    FromFunc->setHasProfileAvailable();1364 1365  int64_t Count = Counters[0];1366  int64_t Mispreds = Counters[1];1367 1368  /// Record basic IP sample into \p BasicSamples and return.1369  if (Type == SAMPLE) {1370    BasicSamples[FromOffset] += Count;1371    NumTotalSamples += Count;1372    return std::error_code();1373  }1374 1375  const uint64_t ToOffset = Addr[1]->Offset;1376  BinaryFunction *ToFunc = getBinaryFunctionContainingAddress(ToOffset);1377  if (ToFunc)1378    ToFunc->setHasProfileAvailable();1379 1380  /// For fall-through types, adjust locations to match Trace container.1381  if (Type == FT || Type == FT_EXTERNAL_ORIGIN || Type == FT_EXTERNAL_RETURN) {1382    Addr[2] = Location(Addr[1]->Offset); // Trace To1383    Addr[1] = Location(Addr[0]->Offset); // Trace From1384    // Put a magic value into Trace Branch to differentiate from a full trace:1385    if (Type == FT)1386      Addr[0] = Location(Trace::FT_ONLY);1387    else if (Type == FT_EXTERNAL_ORIGIN)1388      Addr[0] = Location(Trace::FT_EXTERNAL_ORIGIN);1389    else if (Type == FT_EXTERNAL_RETURN)1390      Addr[0] = Location(Trace::FT_EXTERNAL_RETURN);1391    else1392      llvm_unreachable("Unexpected fall-through type");1393  }1394 1395  /// For branch type, mark Trace To to differentiate from a full trace.1396  if (Type == BRANCH)1397    Addr[2] = Location(Trace::BR_ONLY);1398 1399  if (Type == RETURN) {1400    if (!Addr[0]->Offset)1401      Addr[0]->Offset = Trace::FT_EXTERNAL_RETURN;1402    else1403      Returns.emplace(Addr[0]->Offset, true);1404  }1405 1406  /// Record a trace.1407  Trace T{Addr[0]->Offset, Addr[1]->Offset, Addr[2]->Offset};1408  TakenBranchInfo TI{(uint64_t)Count, (uint64_t)Mispreds};1409  Traces.emplace_back(T, TI);1410 1411  NumTotalSamples += Count;1412 1413  return std::error_code();1414}1415 1416bool DataAggregator::ignoreKernelInterrupt(LBREntry &LBR) const {1417  return opts::IgnoreInterruptLBR &&1418         (LBR.From >= KernelBaseAddr || LBR.To >= KernelBaseAddr);1419}1420 1421std::error_code DataAggregator::printLBRHeatMap() {1422  outs() << "PERF2BOLT: parse branch events...\n";1423  NamedRegionTimer T("buildHeatmap", "Building heatmap", TimerGroupName,1424                     TimerGroupDesc, opts::TimeAggregator);1425 1426  if (BC->IsLinuxKernel) {1427    opts::HeatmapMaxAddress = 0xffffffffffffffff;1428    opts::HeatmapMinAddress = KernelBaseAddr;1429  }1430  opts::HeatmapBlockSizes &HMBS = opts::HeatmapBlock;1431  Heatmap HM(HMBS[0], opts::HeatmapMinAddress, opts::HeatmapMaxAddress,1432             getTextSections(BC));1433  auto getSymbolValue = [&](const MCSymbol *Symbol) -> uint64_t {1434    if (Symbol)1435      if (ErrorOr<uint64_t> SymValue = BC->getSymbolValue(*Symbol))1436        return SymValue.get();1437    return 0;1438  };1439  HM.HotStart = getSymbolValue(BC->getHotTextStartSymbol());1440  HM.HotEnd = getSymbolValue(BC->getHotTextEndSymbol());1441 1442  if (!NumTotalSamples) {1443    if (opts::BasicAggregation) {1444      errs() << "HEATMAP-ERROR: no basic event samples detected in profile. "1445                "Cannot build heatmap.";1446    } else {1447      errs() << "HEATMAP-ERROR: no brstack traces detected in profile. "1448                "Cannot build heatmap. Use -ba for building heatmap from "1449                "basic events.\n";1450    }1451    exit(1);1452  }1453 1454  outs() << "HEATMAP: building heat map...\n";1455 1456  // Register basic samples and perf LBR addresses not covered by fallthroughs.1457  for (const auto &[PC, Hits] : BasicSamples)1458    HM.registerAddress(PC, Hits);1459  for (const auto &[Trace, Info] : Traces)1460    if (Trace.To != Trace::BR_ONLY)1461      HM.registerAddressRange(Trace.From, Trace.To, Info.TakenCount);1462 1463  if (HM.getNumInvalidRanges())1464    outs() << "HEATMAP: invalid traces: " << HM.getNumInvalidRanges() << '\n';1465 1466  if (!HM.size()) {1467    errs() << "HEATMAP-ERROR: no valid traces registered\n";1468    exit(1);1469  }1470 1471  HM.print(opts::HeatmapOutput);1472  if (opts::HeatmapOutput == "-") {1473    HM.printCDF(opts::HeatmapOutput);1474    HM.printSectionHotness(opts::HeatmapOutput);1475  } else {1476    HM.printCDF(opts::HeatmapOutput + ".csv");1477    HM.printSectionHotness(opts::HeatmapOutput + "-section-hotness.csv");1478  }1479  // Provide coarse-grained heatmaps if requested via zoom-out scales1480  for (const uint64_t NewBucketSize : ArrayRef(HMBS).drop_front()) {1481    HM.resizeBucket(NewBucketSize);1482    if (opts::HeatmapOutput == "-")1483      HM.print(opts::HeatmapOutput);1484    else1485      HM.print(formatv("{0}-{1}", opts::HeatmapOutput, NewBucketSize).str());1486  }1487 1488  return std::error_code();1489}1490 1491void DataAggregator::parseLBRSample(const PerfBranchSample &Sample,1492                                    bool NeedsSkylakeFix) {1493  // LBRs are stored in reverse execution order. NextLBR refers to the next1494  // executed branch record.1495  const LBREntry *NextLBR = nullptr;1496  uint32_t NumEntry = 0;1497  for (const LBREntry &LBR : Sample.LBR) {1498    ++NumEntry;1499    // Hardware bug workaround: Intel Skylake (which has 32 LBR entries)1500    // sometimes record entry 32 as an exact copy of entry 31. This will cause1501    // us to likely record an invalid trace and generate a stale function for1502    // BAT mode (non BAT disassembles the function and is able to ignore this1503    // trace at aggregation time). Drop first 2 entries (last two, in1504    // chronological order)1505    if (NeedsSkylakeFix && NumEntry <= 2)1506      continue;1507    uint64_t TraceTo = NextLBR ? NextLBR->From : Trace::BR_ONLY;1508    NextLBR = &LBR;1509 1510    TakenBranchInfo &Info = TraceMap[Trace{LBR.From, LBR.To, TraceTo}];1511    ++Info.TakenCount;1512    Info.MispredCount += LBR.Mispred;1513  }1514  // Record LBR addresses not covered by fallthroughs (bottom-of-stack source1515  // and top-of-stack target) as basic samples for heatmap.1516  if (opts::HeatmapMode == opts::HeatmapModeKind::HM_Exclusive &&1517      !Sample.LBR.empty()) {1518    ++BasicSamples[Sample.LBR.front().To];1519    ++BasicSamples[Sample.LBR.back().From];1520  }1521}1522 1523void DataAggregator::printLongRangeTracesDiagnostic() const {1524  outs() << "PERF2BOLT: out of range traces involving unknown regions: "1525         << NumLongRangeTraces;1526  if (NumTraces > 0)1527    outs() << format(" (%.1f%%)", NumLongRangeTraces * 100.0f / NumTraces);1528  outs() << "\n";1529}1530 1531static float printColoredPct(uint64_t Numerator, uint64_t Denominator, float T1,1532                             float T2) {1533  if (Denominator == 0) {1534    outs() << "\n";1535    return 0;1536  }1537  float Percent = Numerator * 100.0f / Denominator;1538  outs() << " (";1539  if (outs().has_colors()) {1540    if (Percent > T2)1541      outs().changeColor(raw_ostream::RED);1542    else if (Percent > T1)1543      outs().changeColor(raw_ostream::YELLOW);1544    else1545      outs().changeColor(raw_ostream::GREEN);1546  }1547  outs() << format("%.1f%%", Percent);1548  if (outs().has_colors())1549    outs().resetColor();1550  outs() << ")\n";1551  return Percent;1552}1553 1554void DataAggregator::printBranchSamplesDiagnostics() const {1555  outs() << "PERF2BOLT: traces mismatching disassembled function contents: "1556         << NumInvalidTraces;1557  if (printColoredPct(NumInvalidTraces, NumTraces, 5, 10) > 10)1558    outs() << "\n !! WARNING !! This high mismatch ratio indicates the input "1559              "binary is probably not the same binary used during profiling "1560              "collection. The generated data may be ineffective for improving "1561              "performance\n\n";1562  printLongRangeTracesDiagnostic();1563}1564 1565void DataAggregator::printBasicSamplesDiagnostics(1566    uint64_t OutOfRangeSamples) const {1567  outs() << "PERF2BOLT: out of range samples recorded in unknown regions: "1568         << OutOfRangeSamples;1569  if (printColoredPct(OutOfRangeSamples, NumTotalSamples, 40, 60) > 80)1570    outs() << "\n !! WARNING !! This high mismatch ratio indicates the input "1571              "binary is probably not the same binary used during profiling "1572              "collection. The generated data may be ineffective for improving "1573              "performance\n\n";1574}1575 1576void DataAggregator::printBranchStacksDiagnostics(1577    uint64_t IgnoredSamples) const {1578  outs() << "PERF2BOLT: ignored samples: " << IgnoredSamples;1579  if (printColoredPct(IgnoredSamples, NumTotalSamples, 20, 50) > 50)1580    errs() << "PERF2BOLT-WARNING: less than 50% of all recorded samples "1581              "were attributed to the input binary\n";1582}1583 1584std::error_code DataAggregator::parseBranchEvents() {1585  std::string BranchEventTypeStr =1586      opts::ArmSPE ? "SPE branch events in brstack-format" : "branch events";1587  outs() << "PERF2BOLT: parse " << BranchEventTypeStr << "...\n";1588  NamedRegionTimer T("parseBranch", "Parsing branch events", TimerGroupName,1589                     TimerGroupDesc, opts::TimeAggregator);1590 1591  uint64_t NumEntries = 0;1592  uint64_t NumSamples = 0;1593  uint64_t NumSamplesNoLBR = 0;1594  bool NeedsSkylakeFix = false;1595 1596  while (hasData() && NumTotalSamples < opts::MaxSamples) {1597    ++NumTotalSamples;1598 1599    ErrorOr<PerfBranchSample> SampleRes = parseBranchSample();1600    if (std::error_code EC = SampleRes.getError()) {1601      if (EC == errc::no_such_process)1602        continue;1603      return EC;1604    }1605    ++NumSamples;1606 1607    PerfBranchSample &Sample = SampleRes.get();1608 1609    if (Sample.LBR.empty()) {1610      ++NumSamplesNoLBR;1611      continue;1612    }1613 1614    NumEntries += Sample.LBR.size();1615    if (this->BC->isX86() && BAT && Sample.LBR.size() == 32 &&1616        !NeedsSkylakeFix) {1617      errs() << "PERF2BOLT-WARNING: using Intel Skylake bug workaround\n";1618      NeedsSkylakeFix = true;1619    }1620 1621    parseLBRSample(Sample, NeedsSkylakeFix);1622  }1623 1624  Traces.reserve(TraceMap.size());1625  for (const auto &[Trace, Info] : TraceMap) {1626    Traces.emplace_back(Trace, Info);1627    for (const uint64_t Addr : {Trace.Branch, Trace.From})1628      if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Addr))1629        BF->setHasProfileAvailable();1630  }1631  clear(TraceMap);1632 1633  outs() << "PERF2BOLT: read " << NumSamples << " samples and " << NumEntries1634         << " brstack entries\n";1635  if (NumTotalSamples) {1636    if (NumSamples && NumSamplesNoLBR == NumSamples) {1637      // Note: we don't know if perf2bolt is being used to parse memory samples1638      // at this point. In this case, it is OK to parse zero LBRs.1639      if (!opts::ArmSPE)1640        errs()1641            << "PERF2BOLT-WARNING: all recorded samples for this binary lack "1642               "brstack. Record profile with perf record -j any or run "1643               "perf2bolt "1644               "in non-brstack mode with -ba (the performance improvement in "1645               "-ba "1646               "mode may be limited)\n";1647      else1648        errs()1649            << "PERF2BOLT-WARNING: All recorded samples for this binary lack "1650               "SPE brstack entries. Make sure you are running Linux perf 6.14 "1651               "or later, otherwise you get zero samples. Record the profile "1652               "with: perf record -e 'arm_spe_0/branch_filter=1/'.";1653    } else {1654      printBranchStacksDiagnostics(NumTotalSamples - NumSamples);1655    }1656  }1657 1658  return std::error_code();1659}1660 1661void DataAggregator::processBranchEvents() {1662  outs() << "PERF2BOLT: processing branch events...\n";1663  NamedRegionTimer T("processBranch", "Processing branch events",1664                     TimerGroupName, TimerGroupDesc, opts::TimeAggregator);1665 1666  Returns.emplace(Trace::FT_EXTERNAL_RETURN, true);1667  for (const auto &[Trace, Info] : Traces) {1668    bool IsReturn = checkReturn(Trace.Branch);1669    // Ignore returns.1670    if (!IsReturn && Trace.Branch != Trace::FT_ONLY &&1671        Trace.Branch != Trace::FT_EXTERNAL_ORIGIN)1672      doBranch(Trace.Branch, Trace.From, Info.TakenCount, Info.MispredCount);1673    if (Trace.To != Trace::BR_ONLY)1674      doTrace(Trace, Info.TakenCount, IsReturn);1675  }1676  printBranchSamplesDiagnostics();1677}1678 1679std::error_code DataAggregator::parseBasicEvents() {1680  outs() << "PERF2BOLT: parsing basic events (without brstack)...\n";1681  NamedRegionTimer T("parseBasic", "Parsing basic events", TimerGroupName,1682                     TimerGroupDesc, opts::TimeAggregator);1683  while (hasData()) {1684    ErrorOr<PerfBasicSample> Sample = parseBasicSample();1685    if (std::error_code EC = Sample.getError())1686      return EC;1687 1688    if (!Sample->PC)1689      continue;1690    ++NumTotalSamples;1691 1692    if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Sample->PC))1693      BF->setHasProfileAvailable();1694 1695    ++BasicSamples[Sample->PC];1696    EventNames.insert(Sample->EventName);1697  }1698  outs() << "PERF2BOLT: read " << NumTotalSamples << " basic samples\n";1699 1700  return std::error_code();1701}1702 1703void DataAggregator::processBasicEvents() {1704  outs() << "PERF2BOLT: processing basic events (without brstack)...\n";1705  NamedRegionTimer T("processBasic", "Processing basic events", TimerGroupName,1706                     TimerGroupDesc, opts::TimeAggregator);1707  uint64_t OutOfRangeSamples = 0;1708  for (auto &Sample : BasicSamples) {1709    const uint64_t PC = Sample.first;1710    const uint64_t HitCount = Sample.second;1711    BinaryFunction *Func = getBinaryFunctionContainingAddress(PC);1712    if (!Func) {1713      OutOfRangeSamples += HitCount;1714      continue;1715    }1716 1717    doBasicSample(*Func, PC, HitCount);1718  }1719 1720  printBasicSamplesDiagnostics(OutOfRangeSamples);1721}1722 1723std::error_code DataAggregator::parseMemEvents() {1724  outs() << "PERF2BOLT: parsing memory events...\n";1725  NamedRegionTimer T("parseMemEvents", "Parsing mem events", TimerGroupName,1726                     TimerGroupDesc, opts::TimeAggregator);1727  while (hasData()) {1728    ErrorOr<PerfMemSample> Sample = parseMemSample();1729    if (std::error_code EC = Sample.getError())1730      return EC;1731 1732    if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Sample->PC))1733      BF->setHasProfileAvailable();1734 1735    MemSamples.emplace_back(std::move(Sample.get()));1736  }1737 1738  return std::error_code();1739}1740 1741void DataAggregator::processMemEvents() {1742  NamedRegionTimer T("ProcessMemEvents", "Processing mem events",1743                     TimerGroupName, TimerGroupDesc, opts::TimeAggregator);1744  for (const PerfMemSample &Sample : MemSamples) {1745    uint64_t PC = Sample.PC;1746    uint64_t Addr = Sample.Addr;1747    StringRef FuncName;1748    StringRef MemName;1749 1750    // Try to resolve symbol for PC1751    BinaryFunction *Func = getBinaryFunctionContainingAddress(PC);1752    if (!Func) {1753      LLVM_DEBUG(if (PC != 0) {1754        dbgs() << formatv("Skipped mem event: {0:x} => {1:x}\n", PC, Addr);1755      });1756      continue;1757    }1758 1759    FuncName = Func->getOneName();1760    PC -= Func->getAddress();1761 1762    // Try to resolve symbol for memory load1763    if (BinaryData *BD = BC->getBinaryDataContainingAddress(Addr)) {1764      MemName = BD->getName();1765      Addr -= BD->getAddress();1766    } else if (opts::FilterMemProfile) {1767      // Filter out heap/stack accesses1768      continue;1769    }1770 1771    const Location FuncLoc(!FuncName.empty(), FuncName, PC);1772    const Location AddrLoc(!MemName.empty(), MemName, Addr);1773 1774    FuncMemData *MemData = &NamesToMemEvents[FuncName];1775    MemData->Name = FuncName;1776    setMemData(*Func, MemData);1777    MemData->update(FuncLoc, AddrLoc);1778    LLVM_DEBUG(dbgs() << "Mem event: " << FuncLoc << " = " << AddrLoc << "\n");1779  }1780}1781 1782std::error_code DataAggregator::parsePreAggregatedLBRSamples() {1783  outs() << "PERF2BOLT: parsing pre-aggregated profile...\n";1784  NamedRegionTimer T("parseAggregated", "Parsing aggregated branch events",1785                     TimerGroupName, TimerGroupDesc, opts::TimeAggregator);1786  size_t AggregatedLBRs = 0;1787  while (hasData()) {1788    if (std::error_code EC = parseAggregatedLBREntry())1789      return EC;1790    ++AggregatedLBRs;1791  }1792 1793  outs() << "PERF2BOLT: read " << AggregatedLBRs1794         << " aggregated brstack entries\n";1795 1796  return std::error_code();1797}1798 1799std::optional<int32_t> DataAggregator::parseCommExecEvent() {1800  size_t LineEnd = ParsingBuf.find_first_of("\n");1801  if (LineEnd == StringRef::npos) {1802    reportError("expected rest of line");1803    Diag << "Found: " << ParsingBuf << "\n";1804    return std::nullopt;1805  }1806  StringRef Line = ParsingBuf.substr(0, LineEnd);1807 1808  size_t Pos = Line.find("PERF_RECORD_COMM exec");1809  if (Pos == StringRef::npos)1810    return std::nullopt;1811  Line = Line.drop_front(Pos);1812 1813  // Line:1814  //  PERF_RECORD_COMM exec: <name>:<pid>/<tid>"1815  StringRef PIDStr = Line.rsplit(':').second.split('/').first;1816  int32_t PID;1817  if (PIDStr.getAsInteger(10, PID)) {1818    reportError("expected PID");1819    Diag << "Found: " << PIDStr << "in '" << Line << "'\n";1820    return std::nullopt;1821  }1822 1823  return PID;1824}1825 1826namespace {1827std::optional<uint64_t> parsePerfTime(const StringRef TimeStr) {1828  const StringRef SecTimeStr = TimeStr.split('.').first;1829  const StringRef USecTimeStr = TimeStr.split('.').second;1830  uint64_t SecTime;1831  uint64_t USecTime;1832  if (SecTimeStr.getAsInteger(10, SecTime) ||1833      USecTimeStr.getAsInteger(10, USecTime))1834    return std::nullopt;1835  return SecTime * 1000000ULL + USecTime;1836}1837}1838 1839std::optional<DataAggregator::ForkInfo> DataAggregator::parseForkEvent() {1840  while (checkAndConsumeFS()) {1841  }1842 1843  size_t LineEnd = ParsingBuf.find_first_of("\n");1844  if (LineEnd == StringRef::npos) {1845    reportError("expected rest of line");1846    Diag << "Found: " << ParsingBuf << "\n";1847    return std::nullopt;1848  }1849  StringRef Line = ParsingBuf.substr(0, LineEnd);1850 1851  size_t Pos = Line.find("PERF_RECORD_FORK");1852  if (Pos == StringRef::npos) {1853    consumeRestOfLine();1854    return std::nullopt;1855  }1856 1857  ForkInfo FI;1858 1859  const StringRef TimeStr =1860      Line.substr(0, Pos).rsplit(':').first.rsplit(FieldSeparator).second;1861  if (std::optional<uint64_t> TimeRes = parsePerfTime(TimeStr)) {1862    FI.Time = *TimeRes;1863  }1864 1865  Line = Line.drop_front(Pos);1866 1867  // Line:1868  //  PERF_RECORD_FORK(<child_pid>:<child_tid>):(<parent_pid>:<parent_tid>)1869  const StringRef ChildPIDStr = Line.split('(').second.split(':').first;1870  if (ChildPIDStr.getAsInteger(10, FI.ChildPID)) {1871    reportError("expected PID");1872    Diag << "Found: " << ChildPIDStr << "in '" << Line << "'\n";1873    return std::nullopt;1874  }1875 1876  const StringRef ParentPIDStr = Line.rsplit('(').second.split(':').first;1877  if (ParentPIDStr.getAsInteger(10, FI.ParentPID)) {1878    reportError("expected PID");1879    Diag << "Found: " << ParentPIDStr << "in '" << Line << "'\n";1880    return std::nullopt;1881  }1882 1883  consumeRestOfLine();1884 1885  return FI;1886}1887 1888ErrorOr<std::pair<StringRef, DataAggregator::MMapInfo>>1889DataAggregator::parseMMapEvent() {1890  while (checkAndConsumeFS()) {1891  }1892 1893  MMapInfo ParsedInfo;1894 1895  size_t LineEnd = ParsingBuf.find_first_of("\n");1896  if (LineEnd == StringRef::npos) {1897    reportError("expected rest of line");1898    Diag << "Found: " << ParsingBuf << "\n";1899    return make_error_code(llvm::errc::io_error);1900  }1901  StringRef Line = ParsingBuf.substr(0, LineEnd);1902 1903  size_t Pos = Line.find("PERF_RECORD_MMAP2");1904  if (Pos == StringRef::npos) {1905    consumeRestOfLine();1906    return std::make_pair(StringRef(), ParsedInfo);1907  }1908 1909  // Line:1910  //   {<name> .* <sec>.<usec>: }PERF_RECORD_MMAP2 <pid>/<tid>: .* <file_name>1911 1912  const StringRef TimeStr =1913      Line.substr(0, Pos).rsplit(':').first.rsplit(FieldSeparator).second;1914  if (std::optional<uint64_t> TimeRes = parsePerfTime(TimeStr))1915    ParsedInfo.Time = *TimeRes;1916 1917  Line = Line.drop_front(Pos);1918 1919  // Line:1920  //   PERF_RECORD_MMAP2 <pid>/<tid>: [<hexbase>(<hexsize>) .*]: .* <file_name>1921 1922  StringRef FileName = Line.rsplit(FieldSeparator).second;1923  if (FileName.starts_with("//") || FileName.starts_with("[")) {1924    consumeRestOfLine();1925    return std::make_pair(StringRef(), ParsedInfo);1926  }1927  FileName = sys::path::filename(FileName);1928 1929  const StringRef PIDStr = Line.split(FieldSeparator).second.split('/').first;1930  if (PIDStr.getAsInteger(10, ParsedInfo.PID)) {1931    reportError("expected PID");1932    Diag << "Found: " << PIDStr << "in '" << Line << "'\n";1933    return make_error_code(llvm::errc::io_error);1934  }1935 1936  const StringRef BaseAddressStr = Line.split('[').second.split('(').first;1937  if (BaseAddressStr.getAsInteger(0, ParsedInfo.MMapAddress)) {1938    reportError("expected base address");1939    Diag << "Found: " << BaseAddressStr << "in '" << Line << "'\n";1940    return make_error_code(llvm::errc::io_error);1941  }1942 1943  const StringRef SizeStr = Line.split('(').second.split(')').first;1944  if (SizeStr.getAsInteger(0, ParsedInfo.Size)) {1945    reportError("expected mmaped size");1946    Diag << "Found: " << SizeStr << "in '" << Line << "'\n";1947    return make_error_code(llvm::errc::io_error);1948  }1949 1950  const StringRef OffsetStr =1951      Line.split('@').second.ltrim().split(FieldSeparator).first;1952  if (OffsetStr.getAsInteger(0, ParsedInfo.Offset)) {1953    reportError("expected mmaped page-aligned offset");1954    Diag << "Found: " << OffsetStr << "in '" << Line << "'\n";1955    return make_error_code(llvm::errc::io_error);1956  }1957 1958  consumeRestOfLine();1959 1960  return std::make_pair(FileName, ParsedInfo);1961}1962 1963std::error_code DataAggregator::parseMMapEvents() {1964  outs() << "PERF2BOLT: parsing perf-script mmap events output\n";1965  NamedRegionTimer T("parseMMapEvents", "Parsing mmap events", TimerGroupName,1966                     TimerGroupDesc, opts::TimeAggregator);1967 1968  std::multimap<StringRef, MMapInfo> GlobalMMapInfo;1969  while (hasData()) {1970    ErrorOr<std::pair<StringRef, MMapInfo>> FileMMapInfoRes = parseMMapEvent();1971    if (std::error_code EC = FileMMapInfoRes.getError())1972      return EC;1973 1974    std::pair<StringRef, MMapInfo> FileMMapInfo = FileMMapInfoRes.get();1975    if (FileMMapInfo.second.PID == -1)1976      continue;1977    if (FileMMapInfo.first == "(deleted)")1978      continue;1979 1980    GlobalMMapInfo.insert(FileMMapInfo);1981  }1982 1983  LLVM_DEBUG({1984    dbgs() << "FileName -> mmap info:\n"1985           << "  Filename : PID [MMapAddr, Size, Offset]\n";1986    for (const auto &[Name, MMap] : GlobalMMapInfo)1987      dbgs() << formatv("  {0} : {1} [{2:x}, {3:x} @ {4:x}]\n", Name, MMap.PID,1988                        MMap.MMapAddress, MMap.Size, MMap.Offset);1989  });1990 1991  StringRef NameToUse = llvm::sys::path::filename(BC->getFilename());1992  if (GlobalMMapInfo.count(NameToUse) == 0 && !BuildIDBinaryName.empty()) {1993    errs() << "PERF2BOLT-WARNING: using \"" << BuildIDBinaryName1994           << "\" for profile matching\n";1995    NameToUse = BuildIDBinaryName;1996  }1997 1998  auto Range = GlobalMMapInfo.equal_range(NameToUse);1999  for (MMapInfo &MMapInfo : llvm::make_second_range(make_range(Range))) {2000    if (BC->HasFixedLoadAddress && MMapInfo.MMapAddress) {2001      // Check that the binary mapping matches one of the segments.2002      bool MatchFound = llvm::any_of(2003          llvm::make_second_range(BC->SegmentMapInfo),2004          [&](SegmentInfo &SegInfo) {2005            // The mapping is page-aligned and hence the MMapAddress could be2006            // different from the segment start address. We cannot know the page2007            // size of the mapping, but we know it should not exceed the segment2008            // alignment value. Hence we are performing an approximate check.2009            return SegInfo.Address >= MMapInfo.MMapAddress &&2010                   SegInfo.Address - MMapInfo.MMapAddress < SegInfo.Alignment &&2011                   SegInfo.IsExecutable;2012          });2013      if (!MatchFound) {2014        errs() << "PERF2BOLT-WARNING: ignoring mapping of " << NameToUse2015               << " at 0x" << Twine::utohexstr(MMapInfo.MMapAddress) << '\n';2016        continue;2017      }2018    }2019 2020    // Set base address for shared objects.2021    if (!BC->HasFixedLoadAddress) {2022      std::optional<uint64_t> BaseAddress =2023          BC->getBaseAddressForMapping(MMapInfo.MMapAddress, MMapInfo.Offset);2024      if (!BaseAddress) {2025        errs() << "PERF2BOLT-WARNING: unable to find base address of the "2026                  "binary when memory mapped at 0x"2027               << Twine::utohexstr(MMapInfo.MMapAddress)2028               << " using file offset 0x" << Twine::utohexstr(MMapInfo.Offset)2029               << ". Ignoring profile data for this mapping\n";2030        continue;2031      }2032      MMapInfo.BaseAddress = *BaseAddress;2033    }2034 2035    // Try to add MMapInfo to the map and update its size. Large binaries may2036    // span to multiple text segments, so the mapping is inserted only on the2037    // first occurrence.2038    if (!BinaryMMapInfo.insert(std::make_pair(MMapInfo.PID, MMapInfo)).second)2039      assert(MMapInfo.BaseAddress == BinaryMMapInfo[MMapInfo.PID].BaseAddress &&2040             "Base address on multiple segment mappings should match");2041 2042    // Update mapping size.2043    const uint64_t EndAddress = MMapInfo.MMapAddress + MMapInfo.Size;2044    const uint64_t Size = EndAddress - BinaryMMapInfo[MMapInfo.PID].BaseAddress;2045    if (Size > BinaryMMapInfo[MMapInfo.PID].Size)2046      BinaryMMapInfo[MMapInfo.PID].Size = Size;2047  }2048 2049  if (BinaryMMapInfo.empty()) {2050    if (errs().has_colors())2051      errs().changeColor(raw_ostream::RED);2052    errs() << "PERF2BOLT-ERROR: could not find a profile matching binary \""2053           << BC->getFilename() << "\".";2054    if (!GlobalMMapInfo.empty()) {2055      errs() << " Profile for the following binary name(s) is available:\n";2056      for (auto I = GlobalMMapInfo.begin(), IE = GlobalMMapInfo.end(); I != IE;2057           I = GlobalMMapInfo.upper_bound(I->first))2058        errs() << "  " << I->first << '\n';2059      errs() << "Please rename the input binary.\n";2060    } else {2061      errs() << " Failed to extract any binary name from a profile.\n";2062    }2063    if (errs().has_colors())2064      errs().resetColor();2065 2066    exit(1);2067  }2068 2069  return std::error_code();2070}2071 2072std::error_code DataAggregator::parseTaskEvents() {2073  outs() << "PERF2BOLT: parsing perf-script task events output\n";2074  NamedRegionTimer T("parseTaskEvents", "Parsing task events", TimerGroupName,2075                     TimerGroupDesc, opts::TimeAggregator);2076 2077  while (hasData()) {2078    if (std::optional<int32_t> CommInfo = parseCommExecEvent()) {2079      // Remove forked child that ran execve2080      auto MMapInfoIter = BinaryMMapInfo.find(*CommInfo);2081      if (MMapInfoIter != BinaryMMapInfo.end() && MMapInfoIter->second.Forked)2082        BinaryMMapInfo.erase(MMapInfoIter);2083      consumeRestOfLine();2084      continue;2085    }2086 2087    std::optional<ForkInfo> ForkInfo = parseForkEvent();2088    if (!ForkInfo)2089      continue;2090 2091    if (ForkInfo->ParentPID == ForkInfo->ChildPID)2092      continue;2093 2094    if (ForkInfo->Time == 0) {2095      // Process was forked and mmaped before perf ran. In this case the child2096      // should have its own mmap entry unless it was execve'd.2097      continue;2098    }2099 2100    auto MMapInfoIter = BinaryMMapInfo.find(ForkInfo->ParentPID);2101    if (MMapInfoIter == BinaryMMapInfo.end())2102      continue;2103 2104    MMapInfo MMapInfo = MMapInfoIter->second;2105    MMapInfo.PID = ForkInfo->ChildPID;2106    MMapInfo.Forked = true;2107    BinaryMMapInfo.insert(std::make_pair(MMapInfo.PID, MMapInfo));2108  }2109 2110  outs() << "PERF2BOLT: input binary is associated with "2111         << BinaryMMapInfo.size() << " PID(s)\n";2112 2113  LLVM_DEBUG({2114    for (const MMapInfo &MMI : llvm::make_second_range(BinaryMMapInfo))2115      outs() << formatv("  {0}{1}: ({2:x}: {3:x})\n", MMI.PID,2116                        (MMI.Forked ? " (forked)" : ""), MMI.MMapAddress,2117                        MMI.Size);2118  });2119 2120  return std::error_code();2121}2122 2123std::optional<std::pair<StringRef, StringRef>>2124DataAggregator::parseNameBuildIDPair() {2125  while (checkAndConsumeFS()) {2126  }2127 2128  ErrorOr<StringRef> BuildIDStr = parseString(FieldSeparator, true);2129  if (std::error_code EC = BuildIDStr.getError())2130    return std::nullopt;2131 2132  // If one of the strings is missing, don't issue a parsing error, but still2133  // do not return a value.2134  consumeAllRemainingFS();2135  if (checkNewLine())2136    return std::nullopt;2137 2138  ErrorOr<StringRef> NameStr = parseString(FieldSeparator, true);2139  if (std::error_code EC = NameStr.getError())2140    return std::nullopt;2141 2142  consumeRestOfLine();2143  return std::make_pair(NameStr.get(), BuildIDStr.get());2144}2145 2146bool DataAggregator::hasAllBuildIDs() {2147  const StringRef SavedParsingBuf = ParsingBuf;2148 2149  if (!hasData())2150    return false;2151 2152  bool HasInvalidEntries = false;2153  while (hasData()) {2154    if (!parseNameBuildIDPair()) {2155      HasInvalidEntries = true;2156      break;2157    }2158  }2159 2160  ParsingBuf = SavedParsingBuf;2161 2162  return !HasInvalidEntries;2163}2164 2165std::optional<StringRef>2166DataAggregator::getFileNameForBuildID(StringRef FileBuildID) {2167  const StringRef SavedParsingBuf = ParsingBuf;2168 2169  StringRef FileName;2170  while (hasData()) {2171    std::optional<std::pair<StringRef, StringRef>> IDPair =2172        parseNameBuildIDPair();2173    if (!IDPair) {2174      consumeRestOfLine();2175      continue;2176    }2177 2178    if (IDPair->second.starts_with(FileBuildID)) {2179      FileName = sys::path::filename(IDPair->first);2180      break;2181    }2182  }2183 2184  ParsingBuf = SavedParsingBuf;2185 2186  if (!FileName.empty())2187    return FileName;2188 2189  return std::nullopt;2190}2191 2192std::error_code2193DataAggregator::writeAggregatedFile(StringRef OutputFilename) const {2194  std::error_code EC;2195  raw_fd_ostream OutFile(OutputFilename, EC, sys::fs::OpenFlags::OF_None);2196  if (EC)2197    return EC;2198 2199  bool WriteMemLocs = false;2200 2201  auto writeLocation = [&OutFile, &WriteMemLocs](const Location &Loc) {2202    if (WriteMemLocs)2203      OutFile << (Loc.IsSymbol ? "4 " : "3 ");2204    else2205      OutFile << (Loc.IsSymbol ? "1 " : "0 ");2206    OutFile << (Loc.Name.empty() ? "[unknown]" : getEscapedName(Loc.Name))2207            << " " << Twine::utohexstr(Loc.Offset) << FieldSeparator;2208  };2209 2210  uint64_t BranchValues = 0;2211  uint64_t MemValues = 0;2212 2213  if (BAT)2214    OutFile << "boltedcollection\n";2215  if (opts::BasicAggregation) {2216    OutFile << "no_lbr";2217    for (const StringMapEntry<EmptyStringSetTag> &Entry : EventNames)2218      OutFile << " " << Entry.getKey();2219    OutFile << "\n";2220 2221    for (const auto &KV : NamesToBasicSamples) {2222      const FuncBasicSampleData &FSD = KV.second;2223      for (const BasicSampleInfo &SI : FSD.Data) {2224        writeLocation(SI.Loc);2225        OutFile << SI.Hits << "\n";2226        ++BranchValues;2227      }2228    }2229  } else {2230    for (const auto &KV : NamesToBranches) {2231      const FuncBranchData &FBD = KV.second;2232      for (const BranchInfo &BI : FBD.Data) {2233        writeLocation(BI.From);2234        writeLocation(BI.To);2235        OutFile << BI.Mispreds << " " << BI.Branches << "\n";2236        ++BranchValues;2237      }2238      for (const BranchInfo &BI : FBD.EntryData) {2239        // Do not output if source is a known symbol, since this was already2240        // accounted for in the source function2241        if (BI.From.IsSymbol)2242          continue;2243        writeLocation(BI.From);2244        writeLocation(BI.To);2245        OutFile << BI.Mispreds << " " << BI.Branches << "\n";2246        ++BranchValues;2247      }2248    }2249 2250    WriteMemLocs = true;2251    for (const auto &KV : NamesToMemEvents) {2252      const FuncMemData &FMD = KV.second;2253      for (const MemInfo &MemEvent : FMD.Data) {2254        writeLocation(MemEvent.Offset);2255        writeLocation(MemEvent.Addr);2256        OutFile << MemEvent.Count << "\n";2257        ++MemValues;2258      }2259    }2260  }2261 2262  outs() << "PERF2BOLT: wrote " << BranchValues << " objects and " << MemValues2263         << " memory objects to " << OutputFilename << "\n";2264 2265  return std::error_code();2266}2267 2268std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,2269                                             StringRef OutputFilename) const {2270  std::error_code EC;2271  raw_fd_ostream OutFile(OutputFilename, EC, sys::fs::OpenFlags::OF_None);2272  if (EC)2273    return EC;2274 2275  yaml::bolt::BinaryProfile BP;2276 2277  const MCPseudoProbeDecoder *PseudoProbeDecoder =2278      opts::ProfileWritePseudoProbes ? BC.getPseudoProbeDecoder() : nullptr;2279 2280  // Fill out the header info.2281  BP.Header.Version = 1;2282  BP.Header.FileName = std::string(BC.getFilename());2283  std::optional<StringRef> BuildID = BC.getFileBuildID();2284  BP.Header.Id = BuildID ? std::string(*BuildID) : "<unknown>";2285  BP.Header.Origin = std::string(getReaderName());2286  // Only the input binary layout order is supported.2287  BP.Header.IsDFSOrder = false;2288  // FIXME: Need to match hash function used to produce BAT hashes.2289  BP.Header.HashFunction = HashFunction::Default;2290 2291  ListSeparator LS(",");2292  raw_string_ostream EventNamesOS(BP.Header.EventNames);2293  for (const StringMapEntry<EmptyStringSetTag> &EventEntry : EventNames)2294    EventNamesOS << LS << EventEntry.first().str();2295 2296  BP.Header.Flags = opts::BasicAggregation ? BinaryFunction::PF_BASIC2297                                           : BinaryFunction::PF_BRANCH;2298 2299  // Add probe inline tree nodes.2300  YAMLProfileWriter::InlineTreeDesc InlineTree;2301  if (PseudoProbeDecoder)2302    std::tie(BP.PseudoProbeDesc, InlineTree) =2303        YAMLProfileWriter::convertPseudoProbeDesc(*PseudoProbeDecoder);2304 2305  if (!opts::BasicAggregation) {2306    // Convert profile for functions not covered by BAT2307    for (auto &BFI : BC.getBinaryFunctions()) {2308      BinaryFunction &Function = BFI.second;2309      if (!Function.hasProfile())2310        continue;2311      if (BAT->isBATFunction(Function.getAddress()))2312        continue;2313      BP.Functions.emplace_back(YAMLProfileWriter::convert(2314          Function, /*UseDFS=*/false, InlineTree, BAT));2315    }2316 2317    for (const auto &KV : NamesToBranches) {2318      const StringRef FuncName = KV.first;2319      const FuncBranchData &Branches = KV.second;2320      yaml::bolt::BinaryFunctionProfile YamlBF;2321      BinaryData *BD = BC.getBinaryDataByName(FuncName);2322      assert(BD);2323      uint64_t FuncAddress = BD->getAddress();2324      if (!BAT->isBATFunction(FuncAddress))2325        continue;2326      BinaryFunction *BF = BC.getBinaryFunctionAtAddress(FuncAddress);2327      assert(BF);2328      YamlBF.Name = getLocationName(*BF, BAT);2329      YamlBF.Id = BF->getFunctionNumber();2330      YamlBF.Hash = BAT->getBFHash(FuncAddress);2331      YamlBF.ExecCount = BF->getKnownExecutionCount();2332      YamlBF.ExternEntryCount = BF->getExternEntryCount();2333      YamlBF.NumBasicBlocks = BAT->getNumBasicBlocks(FuncAddress);2334      const BoltAddressTranslation::BBHashMapTy &BlockMap =2335          BAT->getBBHashMap(FuncAddress);2336      YamlBF.Blocks.resize(YamlBF.NumBasicBlocks);2337 2338      for (auto &&[Entry, YamlBB] : llvm::zip(BlockMap, YamlBF.Blocks)) {2339        const auto &Block = Entry.second;2340        YamlBB.Hash = Block.Hash;2341        YamlBB.Index = Block.Index;2342      }2343 2344      // Lookup containing basic block offset and index2345      auto getBlock = [&BlockMap](uint32_t Offset) {2346        auto BlockIt = BlockMap.upper_bound(Offset);2347        if (LLVM_UNLIKELY(BlockIt == BlockMap.begin())) {2348          errs() << "BOLT-ERROR: invalid BAT section\n";2349          exit(1);2350        }2351        --BlockIt;2352        return std::pair(BlockIt->first, BlockIt->second.Index);2353      };2354 2355      for (const BranchInfo &BI : Branches.Data) {2356        using namespace yaml::bolt;2357        const auto &[BlockOffset, BlockIndex] = getBlock(BI.From.Offset);2358        BinaryBasicBlockProfile &YamlBB = YamlBF.Blocks[BlockIndex];2359        if (BI.To.IsSymbol && BI.To.Name == BI.From.Name && BI.To.Offset != 0) {2360          // Internal branch2361          const unsigned SuccIndex = getBlock(BI.To.Offset).second;2362          auto &SI = YamlBB.Successors.emplace_back(SuccessorInfo{SuccIndex});2363          SI.Count = BI.Branches;2364          SI.Mispreds = BI.Mispreds;2365        } else {2366          // Call2367          const uint32_t Offset = BI.From.Offset - BlockOffset;2368          auto &CSI = YamlBB.CallSites.emplace_back(CallSiteInfo{Offset});2369          CSI.Count = BI.Branches;2370          CSI.Mispreds = BI.Mispreds;2371          if (const BinaryData *BD = BC.getBinaryDataByName(BI.To.Name))2372            YAMLProfileWriter::setCSIDestination(BC, CSI, BD->getSymbol(), BAT,2373                                                 BI.To.Offset);2374        }2375      }2376      // Set entry counts, similar to DataReader::readProfile.2377      for (const BranchInfo &BI : Branches.EntryData) {2378        if (!BlockMap.isInputBlock(BI.To.Offset)) {2379          if (opts::Verbosity >= 1)2380            errs() << "BOLT-WARNING: Unexpected EntryData in " << FuncName2381                   << " at 0x" << Twine::utohexstr(BI.To.Offset) << '\n';2382          continue;2383        }2384        const unsigned BlockIndex = BlockMap.getBBIndex(BI.To.Offset);2385        YamlBF.Blocks[BlockIndex].ExecCount += BI.Branches;2386      }2387      if (PseudoProbeDecoder) {2388        DenseMap<const MCDecodedPseudoProbeInlineTree *, uint32_t>2389            InlineTreeNodeId;2390        if (BF->getGUID()) {2391          std::tie(YamlBF.InlineTree, InlineTreeNodeId) =2392              YAMLProfileWriter::convertBFInlineTree(*PseudoProbeDecoder,2393                                                     InlineTree, BF->getGUID());2394        }2395        // Fetch probes belonging to all fragments2396        const AddressProbesMap &ProbeMap =2397            PseudoProbeDecoder->getAddress2ProbesMap();2398        BinaryFunction::FragmentsSetTy Fragments(BF->Fragments);2399        Fragments.insert(BF);2400        DenseMap<uint32_t, YAMLProfileWriter::BlockProbeCtx> BlockCtx;2401        for (const BinaryFunction *F : Fragments) {2402          const uint64_t FuncAddr = F->getAddress();2403          for (const MCDecodedPseudoProbe &Probe :2404               ProbeMap.find(FuncAddr, FuncAddr + F->getSize())) {2405            const uint32_t OutputAddress = Probe.getAddress();2406            const uint32_t InputOffset = BAT->translate(2407                FuncAddr, OutputAddress - FuncAddr, /*IsBranchSrc=*/true);2408            const auto &[BlockOffset, BlockIndex] = getBlock(InputOffset);2409            BlockCtx[BlockIndex].addBlockProbe(InlineTreeNodeId, Probe,2410                                               InputOffset - BlockOffset);2411          }2412        }2413 2414        for (auto &[Block, Ctx] : BlockCtx)2415          Ctx.finalize(YamlBF.Blocks[Block]);2416      }2417      // Skip printing if there's no profile data2418      llvm::erase_if(2419          YamlBF.Blocks, [](const yaml::bolt::BinaryBasicBlockProfile &YamlBB) {2420            auto HasCount = [](const auto &SI) { return SI.Count; };2421            bool HasAnyCount = YamlBB.ExecCount ||2422                               llvm::any_of(YamlBB.Successors, HasCount) ||2423                               llvm::any_of(YamlBB.CallSites, HasCount);2424            return !HasAnyCount;2425          });2426      BP.Functions.emplace_back(YamlBF);2427    }2428  }2429 2430  // Write the profile.2431  yaml::Output Out(OutFile, nullptr, 0);2432  Out << BP;2433  return std::error_code();2434}2435 2436void DataAggregator::dump() const { DataReader::dump(); }2437 2438void DataAggregator::dump(const PerfBranchSample &Sample) const {2439  Diag << "Sample brstack entries: " << Sample.LBR.size() << "\n";2440  for (const LBREntry &LBR : Sample.LBR)2441    Diag << LBR << '\n';2442}2443 2444void DataAggregator::dump(const PerfMemSample &Sample) const {2445  Diag << "Sample mem entries: " << Sample.PC << ": " << Sample.Addr << "\n";2446}2447