2145 lines · cpp
1//===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the class that reads LLVM sample profiles. It10// supports three file formats: text, binary and gcov.11//12// The textual representation is useful for debugging and testing purposes. The13// binary representation is more compact, resulting in smaller file sizes.14//15// The gcov encoding is the one generated by GCC's AutoFDO profile creation16// tool (https://github.com/google/autofdo)17//18// All three encodings can be used interchangeably as an input sample profile.19//20//===----------------------------------------------------------------------===//21 22#include "llvm/ProfileData/SampleProfReader.h"23#include "llvm/ADT/DenseMap.h"24#include "llvm/ADT/STLExtras.h"25#include "llvm/ADT/StringRef.h"26#include "llvm/IR/Module.h"27#include "llvm/IR/ProfileSummary.h"28#include "llvm/ProfileData/ProfileCommon.h"29#include "llvm/ProfileData/SampleProf.h"30#include "llvm/Support/CommandLine.h"31#include "llvm/Support/Compression.h"32#include "llvm/Support/ErrorOr.h"33#include "llvm/Support/JSON.h"34#include "llvm/Support/LEB128.h"35#include "llvm/Support/LineIterator.h"36#include "llvm/Support/MD5.h"37#include "llvm/Support/MemoryBuffer.h"38#include "llvm/Support/VirtualFileSystem.h"39#include "llvm/Support/raw_ostream.h"40#include <algorithm>41#include <cstddef>42#include <cstdint>43#include <limits>44#include <memory>45#include <system_error>46#include <vector>47 48using namespace llvm;49using namespace sampleprof;50 51#define DEBUG_TYPE "samplepgo-reader"52 53// This internal option specifies if the profile uses FS discriminators.54// It only applies to text, and binary format profiles.55// For ext-binary format profiles, the flag is set in the summary.56static cl::opt<bool> ProfileIsFSDisciminator(57 "profile-isfs", cl::Hidden, cl::init(false),58 cl::desc("Profile uses flow sensitive discriminators"));59 60/// Dump the function profile for \p FName.61///62/// \param FContext Name + context of the function to print.63/// \param OS Stream to emit the output to.64void SampleProfileReader::dumpFunctionProfile(const FunctionSamples &FS,65 raw_ostream &OS) {66 OS << "Function: " << FS.getContext().toString() << ": " << FS;67}68 69/// Dump all the function profiles found on stream \p OS.70void SampleProfileReader::dump(raw_ostream &OS) {71 std::vector<NameFunctionSamples> V;72 sortFuncProfiles(Profiles, V);73 for (const auto &I : V)74 dumpFunctionProfile(*I.second, OS);75}76 77static void dumpFunctionProfileJson(const FunctionSamples &S,78 json::OStream &JOS, bool TopLevel = false) {79 auto DumpBody = [&](const BodySampleMap &BodySamples) {80 for (const auto &I : BodySamples) {81 const LineLocation &Loc = I.first;82 const SampleRecord &Sample = I.second;83 JOS.object([&] {84 JOS.attribute("line", Loc.LineOffset);85 if (Loc.Discriminator)86 JOS.attribute("discriminator", Loc.Discriminator);87 JOS.attribute("samples", Sample.getSamples());88 89 auto CallTargets = Sample.getSortedCallTargets();90 if (!CallTargets.empty()) {91 JOS.attributeArray("calls", [&] {92 for (const auto &J : CallTargets) {93 JOS.object([&] {94 JOS.attribute("function", J.first.str());95 JOS.attribute("samples", J.second);96 });97 }98 });99 }100 });101 }102 };103 104 auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) {105 for (const auto &I : CallsiteSamples)106 for (const auto &FS : I.second) {107 const LineLocation &Loc = I.first;108 const FunctionSamples &CalleeSamples = FS.second;109 JOS.object([&] {110 JOS.attribute("line", Loc.LineOffset);111 if (Loc.Discriminator)112 JOS.attribute("discriminator", Loc.Discriminator);113 JOS.attributeArray(114 "samples", [&] { dumpFunctionProfileJson(CalleeSamples, JOS); });115 });116 }117 };118 119 JOS.object([&] {120 JOS.attribute("name", S.getFunction().str());121 JOS.attribute("total", S.getTotalSamples());122 if (TopLevel)123 JOS.attribute("head", S.getHeadSamples());124 125 const auto &BodySamples = S.getBodySamples();126 if (!BodySamples.empty())127 JOS.attributeArray("body", [&] { DumpBody(BodySamples); });128 129 const auto &CallsiteSamples = S.getCallsiteSamples();130 if (!CallsiteSamples.empty())131 JOS.attributeArray("callsites",132 [&] { DumpCallsiteSamples(CallsiteSamples); });133 });134}135 136/// Dump all the function profiles found on stream \p OS in the JSON format.137void SampleProfileReader::dumpJson(raw_ostream &OS) {138 std::vector<NameFunctionSamples> V;139 sortFuncProfiles(Profiles, V);140 json::OStream JOS(OS, 2);141 JOS.arrayBegin();142 for (const auto &F : V)143 dumpFunctionProfileJson(*F.second, JOS, true);144 JOS.arrayEnd();145 146 // Emit a newline character at the end as json::OStream doesn't emit one.147 OS << "\n";148}149 150/// Parse \p Input as function head.151///152/// Parse one line of \p Input, and update function name in \p FName,153/// function's total sample count in \p NumSamples, function's entry154/// count in \p NumHeadSamples.155///156/// \returns true if parsing is successful.157static bool ParseHead(const StringRef &Input, StringRef &FName,158 uint64_t &NumSamples, uint64_t &NumHeadSamples) {159 if (Input[0] == ' ')160 return false;161 size_t n2 = Input.rfind(':');162 size_t n1 = Input.rfind(':', n2 - 1);163 FName = Input.substr(0, n1);164 if (Input.substr(n1 + 1, n2 - n1 - 1).getAsInteger(10, NumSamples))165 return false;166 if (Input.substr(n2 + 1).getAsInteger(10, NumHeadSamples))167 return false;168 return true;169}170 171/// Returns true if line offset \p L is legal (only has 16 bits).172static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; }173 174/// Parse \p Input that contains metadata.175/// Possible metadata:176/// - CFG Checksum information:177/// !CFGChecksum: 12345178/// - CFG Checksum information:179/// !Attributes: 1180/// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash.181static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash,182 uint32_t &Attributes) {183 if (Input.starts_with("!CFGChecksum:")) {184 StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim();185 return !CFGInfo.getAsInteger(10, FunctionHash);186 }187 188 if (Input.starts_with("!Attributes:")) {189 StringRef Attrib = Input.substr(strlen("!Attributes:")).trim();190 return !Attrib.getAsInteger(10, Attributes);191 }192 193 return false;194}195 196enum class LineType {197 CallSiteProfile,198 BodyProfile,199 Metadata,200 VirtualCallTypeProfile,201};202 203// Parse `Input` as a white-space separated list of `vtable:count` pairs. An204// example input line is `_ZTVbar:1471 _ZTVfoo:630`.205static bool parseTypeCountMap(StringRef Input,206 DenseMap<StringRef, uint64_t> &TypeCountMap) {207 for (size_t Index = Input.find_first_not_of(' '); Index != StringRef::npos;) {208 size_t ColonIndex = Input.find(':', Index);209 if (ColonIndex == StringRef::npos)210 return false; // No colon found, invalid format.211 StringRef TypeName = Input.substr(Index, ColonIndex - Index);212 // CountIndex is the start index of count.213 size_t CountStartIndex = ColonIndex + 1;214 // NextIndex is the start index after the 'target:count' pair.215 size_t NextIndex = Input.find_first_of(' ', CountStartIndex);216 uint64_t Count;217 if (Input.substr(CountStartIndex, NextIndex - CountStartIndex)218 .getAsInteger(10, Count))219 return false; // Invalid count.220 // Error on duplicated type names in one line of input.221 auto [Iter, Inserted] = TypeCountMap.insert({TypeName, Count});222 if (!Inserted)223 return false;224 Index = (NextIndex == StringRef::npos)225 ? StringRef::npos226 : Input.find_first_not_of(' ', NextIndex);227 }228 return true;229}230 231/// Parse \p Input as line sample.232///233/// \param Input input line.234/// \param LineTy Type of this line.235/// \param Depth the depth of the inline stack.236/// \param NumSamples total samples of the line/inlined callsite.237/// \param LineOffset line offset to the start of the function.238/// \param Discriminator discriminator of the line.239/// \param TargetCountMap map from indirect call target to count.240/// \param FunctionHash the function's CFG hash, used by pseudo probe.241///242/// returns true if parsing is successful.243static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth,244 uint64_t &NumSamples, uint32_t &LineOffset,245 uint32_t &Discriminator, StringRef &CalleeName,246 DenseMap<StringRef, uint64_t> &TargetCountMap,247 DenseMap<StringRef, uint64_t> &TypeCountMap,248 uint64_t &FunctionHash, uint32_t &Attributes,249 bool &IsFlat) {250 for (Depth = 0; Input[Depth] == ' '; Depth++)251 ;252 if (Depth == 0)253 return false;254 255 if (Input[Depth] == '!') {256 LineTy = LineType::Metadata;257 // This metadata is only for manual inspection only. We already created a258 // FunctionSamples and put it in the profile map, so there is no point259 // to skip profiles even they have no use for ThinLTO.260 if (Input == StringRef(" !Flat")) {261 IsFlat = true;262 return true;263 }264 return parseMetadata(Input.substr(Depth), FunctionHash, Attributes);265 }266 267 size_t n1 = Input.find(':');268 StringRef Loc = Input.substr(Depth, n1 - Depth);269 size_t n2 = Loc.find('.');270 if (n2 == StringRef::npos) {271 if (Loc.getAsInteger(10, LineOffset) || !isOffsetLegal(LineOffset))272 return false;273 Discriminator = 0;274 } else {275 if (Loc.substr(0, n2).getAsInteger(10, LineOffset))276 return false;277 if (Loc.substr(n2 + 1).getAsInteger(10, Discriminator))278 return false;279 }280 281 StringRef Rest = Input.substr(n1 + 2);282 if (isDigit(Rest[0])) {283 LineTy = LineType::BodyProfile;284 size_t n3 = Rest.find(' ');285 if (n3 == StringRef::npos) {286 if (Rest.getAsInteger(10, NumSamples))287 return false;288 } else {289 if (Rest.substr(0, n3).getAsInteger(10, NumSamples))290 return false;291 }292 // Find call targets and their sample counts.293 // Note: In some cases, there are symbols in the profile which are not294 // mangled. To accommodate such cases, use colon + integer pairs as the295 // anchor points.296 // An example:297 // _M_construct<char *>:1000 string_view<std::allocator<char> >:437298 // ":1000" and ":437" are used as anchor points so the string above will299 // be interpreted as300 // target: _M_construct<char *>301 // count: 1000302 // target: string_view<std::allocator<char> >303 // count: 437304 while (n3 != StringRef::npos) {305 n3 += Rest.substr(n3).find_first_not_of(' ');306 Rest = Rest.substr(n3);307 n3 = Rest.find_first_of(':');308 if (n3 == StringRef::npos || n3 == 0)309 return false;310 311 StringRef Target;312 uint64_t count, n4;313 while (true) {314 // Get the segment after the current colon.315 StringRef AfterColon = Rest.substr(n3 + 1);316 // Get the target symbol before the current colon.317 Target = Rest.substr(0, n3);318 // Check if the word after the current colon is an integer.319 n4 = AfterColon.find_first_of(' ');320 n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size();321 StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1);322 if (!WordAfterColon.getAsInteger(10, count))323 break;324 325 // Try to find the next colon.326 uint64_t n5 = AfterColon.find_first_of(':');327 if (n5 == StringRef::npos)328 return false;329 n3 += n5 + 1;330 }331 332 // An anchor point is found. Save the {target, count} pair333 TargetCountMap[Target] = count;334 if (n4 == Rest.size())335 break;336 // Change n3 to the next blank space after colon + integer pair.337 n3 = n4;338 }339 } else if (Rest.starts_with(kVTableProfPrefix)) {340 LineTy = LineType::VirtualCallTypeProfile;341 return parseTypeCountMap(Rest.substr(strlen(kVTableProfPrefix)),342 TypeCountMap);343 } else {344 LineTy = LineType::CallSiteProfile;345 size_t n3 = Rest.find_last_of(':');346 CalleeName = Rest.substr(0, n3);347 if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples))348 return false;349 }350 return true;351}352 353/// Load samples from a text file.354///355/// See the documentation at the top of the file for an explanation of356/// the expected format.357///358/// \returns true if the file was loaded successfully, false otherwise.359std::error_code SampleProfileReaderText::readImpl() {360 line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#');361 sampleprof_error Result = sampleprof_error::success;362 363 InlineCallStack InlineStack;364 uint32_t TopLevelProbeProfileCount = 0;365 366 // DepthMetadata tracks whether we have processed metadata for the current367 // top-level or nested function profile.368 uint32_t DepthMetadata = 0;369 370 std::vector<SampleContext *> FlatSamples;371 372 ProfileIsFS = ProfileIsFSDisciminator;373 FunctionSamples::ProfileIsFS = ProfileIsFS;374 for (; !LineIt.is_at_eof(); ++LineIt) {375 size_t pos = LineIt->find_first_not_of(' ');376 if (pos == LineIt->npos || (*LineIt)[pos] == '#')377 continue;378 // Read the header of each function.379 //380 // Note that for function identifiers we are actually expecting381 // mangled names, but we may not always get them. This happens when382 // the compiler decides not to emit the function (e.g., it was inlined383 // and removed). In this case, the binary will not have the linkage384 // name for the function, so the profiler will emit the function's385 // unmangled name, which may contain characters like ':' and '>' in its386 // name (member functions, templates, etc).387 //388 // The only requirement we place on the identifier, then, is that it389 // should not begin with a number.390 if ((*LineIt)[0] != ' ') {391 uint64_t NumSamples, NumHeadSamples;392 StringRef FName;393 if (!ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) {394 reportError(LineIt.line_number(),395 "Expected 'mangled_name:NUM:NUM', found " + *LineIt);396 return sampleprof_error::malformed;397 }398 DepthMetadata = 0;399 SampleContext FContext(FName, CSNameTable);400 if (FContext.hasContext())401 ++CSProfileCount;402 FunctionSamples &FProfile = Profiles.create(FContext);403 mergeSampleProfErrors(Result, FProfile.addTotalSamples(NumSamples));404 mergeSampleProfErrors(Result, FProfile.addHeadSamples(NumHeadSamples));405 InlineStack.clear();406 InlineStack.push_back(&FProfile);407 } else {408 uint64_t NumSamples;409 StringRef FName;410 DenseMap<StringRef, uint64_t> TargetCountMap;411 DenseMap<StringRef, uint64_t> TypeCountMap;412 uint32_t Depth, LineOffset, Discriminator;413 LineType LineTy = LineType::BodyProfile;414 uint64_t FunctionHash = 0;415 uint32_t Attributes = 0;416 bool IsFlat = false;417 // TODO: Update ParseLine to return an error code instead of a bool and418 // report it.419 if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset,420 Discriminator, FName, TargetCountMap, TypeCountMap,421 FunctionHash, Attributes, IsFlat)) {422 switch (LineTy) {423 case LineType::Metadata:424 reportError(LineIt.line_number(),425 "Cannot parse metadata: " + *LineIt);426 break;427 case LineType::VirtualCallTypeProfile:428 reportError(LineIt.line_number(),429 "Expected 'vtables [mangled_vtable:NUM]+', found " +430 *LineIt);431 break;432 default:433 reportError(LineIt.line_number(),434 "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +435 *LineIt);436 }437 return sampleprof_error::malformed;438 }439 if (LineTy != LineType::Metadata && Depth == DepthMetadata) {440 // Metadata must be put at the end of a function profile.441 reportError(LineIt.line_number(),442 "Found non-metadata after metadata: " + *LineIt);443 return sampleprof_error::malformed;444 }445 446 // Here we handle FS discriminators.447 Discriminator &= getDiscriminatorMask();448 449 while (InlineStack.size() > Depth) {450 InlineStack.pop_back();451 }452 switch (LineTy) {453 case LineType::CallSiteProfile: {454 FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(455 LineLocation(LineOffset, Discriminator))[FunctionId(FName)];456 FSamples.setFunction(FunctionId(FName));457 mergeSampleProfErrors(Result, FSamples.addTotalSamples(NumSamples));458 InlineStack.push_back(&FSamples);459 DepthMetadata = 0;460 break;461 }462 463 case LineType::VirtualCallTypeProfile: {464 mergeSampleProfErrors(465 Result, InlineStack.back()->addCallsiteVTableTypeProfAt(466 LineLocation(LineOffset, Discriminator), TypeCountMap));467 break;468 }469 470 case LineType::BodyProfile: {471 FunctionSamples &FProfile = *InlineStack.back();472 for (const auto &name_count : TargetCountMap) {473 mergeSampleProfErrors(Result, FProfile.addCalledTargetSamples(474 LineOffset, Discriminator,475 FunctionId(name_count.first),476 name_count.second));477 }478 mergeSampleProfErrors(479 Result,480 FProfile.addBodySamples(LineOffset, Discriminator, NumSamples));481 break;482 }483 case LineType::Metadata: {484 FunctionSamples &FProfile = *InlineStack.back();485 if (FunctionHash) {486 FProfile.setFunctionHash(FunctionHash);487 if (Depth == 1)488 ++TopLevelProbeProfileCount;489 }490 FProfile.getContext().setAllAttributes(Attributes);491 if (Attributes & (uint32_t)ContextShouldBeInlined)492 ProfileIsPreInlined = true;493 DepthMetadata = Depth;494 if (IsFlat) {495 if (Depth == 1)496 FlatSamples.push_back(&FProfile.getContext());497 else498 Ctx.diagnose(DiagnosticInfoSampleProfile(499 Buffer->getBufferIdentifier(), LineIt.line_number(),500 "!Flat may only be used at top level function.", DS_Warning));501 }502 break;503 }504 }505 }506 }507 508 // Honor the option to skip flat functions. Since they are already added to509 // the profile map, remove them all here.510 if (SkipFlatProf)511 for (SampleContext *FlatSample : FlatSamples)512 Profiles.erase(*FlatSample);513 514 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&515 "Cannot have both context-sensitive and regular profile");516 ProfileIsCS = (CSProfileCount > 0);517 assert((TopLevelProbeProfileCount == 0 ||518 TopLevelProbeProfileCount == Profiles.size()) &&519 "Cannot have both probe-based profiles and regular profiles");520 ProfileIsProbeBased = (TopLevelProbeProfileCount > 0);521 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;522 FunctionSamples::ProfileIsCS = ProfileIsCS;523 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined;524 525 if (Result == sampleprof_error::success)526 computeSummary();527 528 return Result;529}530 531bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) {532 bool result = false;533 534 // Check that the first non-comment line is a valid function header.535 line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#');536 if (!LineIt.is_at_eof()) {537 if ((*LineIt)[0] != ' ') {538 uint64_t NumSamples, NumHeadSamples;539 StringRef FName;540 result = ParseHead(*LineIt, FName, NumSamples, NumHeadSamples);541 }542 }543 544 return result;545}546 547template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() {548 unsigned NumBytesRead = 0;549 uint64_t Val = decodeULEB128(Data, &NumBytesRead);550 551 if (Val > std::numeric_limits<T>::max()) {552 std::error_code EC = sampleprof_error::malformed;553 reportError(0, EC.message());554 return EC;555 } else if (Data + NumBytesRead > End) {556 std::error_code EC = sampleprof_error::truncated;557 reportError(0, EC.message());558 return EC;559 }560 561 Data += NumBytesRead;562 return static_cast<T>(Val);563}564 565ErrorOr<StringRef> SampleProfileReaderBinary::readString() {566 StringRef Str(reinterpret_cast<const char *>(Data));567 if (Data + Str.size() + 1 > End) {568 std::error_code EC = sampleprof_error::truncated;569 reportError(0, EC.message());570 return EC;571 }572 573 Data += Str.size() + 1;574 return Str;575}576 577template <typename T>578ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() {579 if (Data + sizeof(T) > End) {580 std::error_code EC = sampleprof_error::truncated;581 reportError(0, EC.message());582 return EC;583 }584 585 using namespace support;586 T Val = endian::readNext<T, llvm::endianness::little>(Data);587 return Val;588}589 590template <typename T>591inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {592 auto Idx = readNumber<size_t>();593 if (std::error_code EC = Idx.getError())594 return EC;595 if (*Idx >= Table.size())596 return sampleprof_error::truncated_name_table;597 return *Idx;598}599 600ErrorOr<FunctionId>601SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {602 auto Idx = readStringIndex(NameTable);603 if (std::error_code EC = Idx.getError())604 return EC;605 if (RetIdx)606 *RetIdx = *Idx;607 return NameTable[*Idx];608}609 610ErrorOr<SampleContextFrames>611SampleProfileReaderBinary::readContextFromTable(size_t *RetIdx) {612 auto ContextIdx = readNumber<size_t>();613 if (std::error_code EC = ContextIdx.getError())614 return EC;615 if (*ContextIdx >= CSNameTable.size())616 return sampleprof_error::truncated_name_table;617 if (RetIdx)618 *RetIdx = *ContextIdx;619 return CSNameTable[*ContextIdx];620}621 622ErrorOr<std::pair<SampleContext, uint64_t>>623SampleProfileReaderBinary::readSampleContextFromTable() {624 SampleContext Context;625 size_t Idx;626 if (ProfileIsCS) {627 auto FContext(readContextFromTable(&Idx));628 if (std::error_code EC = FContext.getError())629 return EC;630 Context = SampleContext(*FContext);631 } else {632 auto FName(readStringFromTable(&Idx));633 if (std::error_code EC = FName.getError())634 return EC;635 Context = SampleContext(*FName);636 }637 // Since MD5SampleContextStart may point to the profile's file data, need to638 // make sure it is reading the same value on big endian CPU.639 uint64_t Hash = support::endian::read64le(MD5SampleContextStart + Idx);640 // Lazy computing of hash value, write back to the table to cache it. Only641 // compute the context's hash value if it is being referenced for the first642 // time.643 if (Hash == 0) {644 assert(MD5SampleContextStart == MD5SampleContextTable.data());645 Hash = Context.getHashCode();646 support::endian::write64le(&MD5SampleContextTable[Idx], Hash);647 }648 return std::make_pair(Context, Hash);649}650 651std::error_code652SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) {653 auto NumVTableTypes = readNumber<uint32_t>();654 if (std::error_code EC = NumVTableTypes.getError())655 return EC;656 657 for (uint32_t I = 0; I < *NumVTableTypes; ++I) {658 auto VTableType(readStringFromTable());659 if (std::error_code EC = VTableType.getError())660 return EC;661 662 auto VTableSamples = readNumber<uint64_t>();663 if (std::error_code EC = VTableSamples.getError())664 return EC;665 // The source profile should not have duplicate vtable records at the same666 // location. In case duplicate vtables are found, reader can emit a warning667 // but continue processing the profile.668 if (!M.insert(std::make_pair(*VTableType, *VTableSamples)).second) {669 Ctx.diagnose(DiagnosticInfoSampleProfile(670 Buffer->getBufferIdentifier(), 0,671 "Duplicate vtable type " + VTableType->str() +672 " at the same location. Additional counters will be ignored.",673 DS_Warning));674 continue;675 }676 }677 return sampleprof_error::success;678}679 680std::error_code681SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) {682 assert(ReadVTableProf &&683 "Cannot read vtable profiles if ReadVTableProf is false");684 685 // Read the vtable type profile for the callsite.686 auto NumCallsites = readNumber<uint32_t>();687 if (std::error_code EC = NumCallsites.getError())688 return EC;689 690 for (uint32_t I = 0; I < *NumCallsites; ++I) {691 auto LineOffset = readNumber<uint64_t>();692 if (std::error_code EC = LineOffset.getError())693 return EC;694 695 if (!isOffsetLegal(*LineOffset))696 return sampleprof_error::illegal_line_offset;697 698 auto Discriminator = readNumber<uint64_t>();699 if (std::error_code EC = Discriminator.getError())700 return EC;701 702 // Here we handle FS discriminators:703 const uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();704 705 if (std::error_code EC = readVTableTypeCountMap(FProfile.getTypeSamplesAt(706 LineLocation(*LineOffset, DiscriminatorVal))))707 return EC;708 }709 return sampleprof_error::success;710}711 712std::error_code713SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {714 auto NumSamples = readNumber<uint64_t>();715 if (std::error_code EC = NumSamples.getError())716 return EC;717 FProfile.addTotalSamples(*NumSamples);718 719 // Read the samples in the body.720 auto NumRecords = readNumber<uint32_t>();721 if (std::error_code EC = NumRecords.getError())722 return EC;723 724 for (uint32_t I = 0; I < *NumRecords; ++I) {725 auto LineOffset = readNumber<uint64_t>();726 if (std::error_code EC = LineOffset.getError())727 return EC;728 729 if (!isOffsetLegal(*LineOffset)) {730 return sampleprof_error::illegal_line_offset;731 }732 733 auto Discriminator = readNumber<uint64_t>();734 if (std::error_code EC = Discriminator.getError())735 return EC;736 737 auto NumSamples = readNumber<uint64_t>();738 if (std::error_code EC = NumSamples.getError())739 return EC;740 741 auto NumCalls = readNumber<uint32_t>();742 if (std::error_code EC = NumCalls.getError())743 return EC;744 745 // Here we handle FS discriminators:746 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();747 748 for (uint32_t J = 0; J < *NumCalls; ++J) {749 auto CalledFunction(readStringFromTable());750 if (std::error_code EC = CalledFunction.getError())751 return EC;752 753 auto CalledFunctionSamples = readNumber<uint64_t>();754 if (std::error_code EC = CalledFunctionSamples.getError())755 return EC;756 757 FProfile.addCalledTargetSamples(*LineOffset, DiscriminatorVal,758 *CalledFunction, *CalledFunctionSamples);759 }760 761 FProfile.addBodySamples(*LineOffset, DiscriminatorVal, *NumSamples);762 }763 764 // Read all the samples for inlined function calls.765 auto NumCallsites = readNumber<uint32_t>();766 if (std::error_code EC = NumCallsites.getError())767 return EC;768 769 for (uint32_t J = 0; J < *NumCallsites; ++J) {770 auto LineOffset = readNumber<uint64_t>();771 if (std::error_code EC = LineOffset.getError())772 return EC;773 774 auto Discriminator = readNumber<uint64_t>();775 if (std::error_code EC = Discriminator.getError())776 return EC;777 778 auto FName(readStringFromTable());779 if (std::error_code EC = FName.getError())780 return EC;781 782 // Here we handle FS discriminators:783 uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();784 785 FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(786 LineLocation(*LineOffset, DiscriminatorVal))[*FName];787 CalleeProfile.setFunction(*FName);788 if (std::error_code EC = readProfile(CalleeProfile))789 return EC;790 }791 792 if (ReadVTableProf)793 return readCallsiteVTableProf(FProfile);794 795 return sampleprof_error::success;796}797 798std::error_code799SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start,800 SampleProfileMap &Profiles) {801 Data = Start;802 auto NumHeadSamples = readNumber<uint64_t>();803 if (std::error_code EC = NumHeadSamples.getError())804 return EC;805 806 auto FContextHash(readSampleContextFromTable());807 if (std::error_code EC = FContextHash.getError())808 return EC;809 810 auto &[FContext, Hash] = *FContextHash;811 // Use the cached hash value for insertion instead of recalculating it.812 auto Res = Profiles.try_emplace(Hash, FContext, FunctionSamples());813 FunctionSamples &FProfile = Res.first->second;814 FProfile.setContext(FContext);815 FProfile.addHeadSamples(*NumHeadSamples);816 817 if (FContext.hasContext())818 CSProfileCount++;819 820 if (std::error_code EC = readProfile(FProfile))821 return EC;822 return sampleprof_error::success;823}824 825std::error_code826SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {827 return readFuncProfile(Start, Profiles);828}829 830std::error_code SampleProfileReaderBinary::readImpl() {831 ProfileIsFS = ProfileIsFSDisciminator;832 FunctionSamples::ProfileIsFS = ProfileIsFS;833 while (Data < End) {834 if (std::error_code EC = readFuncProfile(Data))835 return EC;836 }837 838 return sampleprof_error::success;839}840 841std::error_code SampleProfileReaderExtBinaryBase::readOneSection(842 const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) {843 Data = Start;844 End = Start + Size;845 switch (Entry.Type) {846 case SecProfSummary:847 if (std::error_code EC = readSummary())848 return EC;849 if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial))850 Summary->setPartialProfile(true);851 if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext))852 FunctionSamples::ProfileIsCS = ProfileIsCS = true;853 if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsPreInlined))854 FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true;855 if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator))856 FunctionSamples::ProfileIsFS = ProfileIsFS = true;857 if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagHasVTableTypeProf))858 ReadVTableProf = true;859 break;860 case SecNameTable: {861 bool FixedLengthMD5 =862 hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5);863 bool UseMD5 = hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name);864 // UseMD5 means if THIS section uses MD5, ProfileIsMD5 means if the entire865 // profile uses MD5 for function name matching in IPO passes.866 ProfileIsMD5 = ProfileIsMD5 || UseMD5;867 FunctionSamples::HasUniqSuffix =868 hasSecFlag(Entry, SecNameTableFlags::SecFlagUniqSuffix);869 if (std::error_code EC = readNameTableSec(UseMD5, FixedLengthMD5))870 return EC;871 break;872 }873 case SecCSNameTable: {874 if (std::error_code EC = readCSNameTableSec())875 return EC;876 break;877 }878 case SecLBRProfile:879 ProfileSecRange = std::make_pair(Data, End);880 if (std::error_code EC = readFuncProfiles())881 return EC;882 break;883 case SecFuncOffsetTable:884 // If module is absent, we are using LLVM tools, and need to read all885 // profiles, so skip reading the function offset table.886 if (!M) {887 Data = End;888 } else {889 assert((!ProfileIsCS ||890 hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered)) &&891 "func offset table should always be sorted in CS profile");892 if (std::error_code EC = readFuncOffsetTable())893 return EC;894 }895 break;896 case SecFuncMetadata: {897 ProfileIsProbeBased =898 hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased);899 FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;900 ProfileHasAttribute =901 hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute);902 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute))903 return EC;904 break;905 }906 case SecProfileSymbolList:907 if (std::error_code EC = readProfileSymbolList())908 return EC;909 break;910 default:911 if (std::error_code EC = readCustomSection(Entry))912 return EC;913 break;914 }915 return sampleprof_error::success;916}917 918bool SampleProfileReaderExtBinaryBase::useFuncOffsetList() const {919 // If profile is CS, the function offset section is expected to consist of920 // sequences of contexts in pre-order layout921 // (e.g. [A, A:1 @ B, A:1 @ B:2.3 @ C] [D, D:1 @ E]), so that when a matched922 // context in the module is found, the profiles of all its callees are923 // recursively loaded. A list is needed since the order of profiles matters.924 if (ProfileIsCS)925 return true;926 927 // If the profile is MD5, use the map container to lookup functions in928 // the module. A remapper has no use on MD5 names.929 if (useMD5())930 return false;931 932 // Profile is not MD5 and if a remapper is present, the remapped name of933 // every function needed to be matched against the module, so use the list934 // container since each entry is accessed.935 if (Remapper)936 return true;937 938 // Otherwise use the map container for faster lookup.939 // TODO: If the cardinality of the function offset section is much smaller940 // than the number of functions in the module, using the list container can941 // be always faster, but we need to figure out the constant factor to942 // determine the cutoff.943 return false;944}945 946std::error_code947SampleProfileReaderExtBinaryBase::read(const DenseSet<StringRef> &FuncsToUse,948 SampleProfileMap &Profiles) {949 if (FuncsToUse.empty())950 return sampleprof_error::success;951 952 Data = ProfileSecRange.first;953 End = ProfileSecRange.second;954 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))955 return EC;956 End = Data;957 DenseSet<FunctionSamples *> ProfilesToReadMetadata;958 for (auto FName : FuncsToUse) {959 auto I = Profiles.find(FName);960 if (I != Profiles.end())961 ProfilesToReadMetadata.insert(&I->second);962 }963 964 if (std::error_code EC =965 readFuncMetadata(ProfileHasAttribute, ProfilesToReadMetadata))966 return EC;967 return sampleprof_error::success;968}969 970bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() {971 if (!M)972 return false;973 FuncsToUse.clear();974 for (auto &F : *M)975 FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F));976 return true;977}978 979std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() {980 // If there are more than one function offset section, the profile associated981 // with the previous section has to be done reading before next one is read.982 FuncOffsetTable.clear();983 FuncOffsetList.clear();984 985 auto Size = readNumber<uint64_t>();986 if (std::error_code EC = Size.getError())987 return EC;988 989 bool UseFuncOffsetList = useFuncOffsetList();990 if (UseFuncOffsetList)991 FuncOffsetList.reserve(*Size);992 else993 FuncOffsetTable.reserve(*Size);994 995 for (uint64_t I = 0; I < *Size; ++I) {996 auto FContextHash(readSampleContextFromTable());997 if (std::error_code EC = FContextHash.getError())998 return EC;999 1000 auto &[FContext, Hash] = *FContextHash;1001 auto Offset = readNumber<uint64_t>();1002 if (std::error_code EC = Offset.getError())1003 return EC;1004 1005 if (UseFuncOffsetList)1006 FuncOffsetList.emplace_back(FContext, *Offset);1007 else1008 // Because Porfiles replace existing value with new value if collision1009 // happens, we also use the latest offset so that they are consistent.1010 FuncOffsetTable[Hash] = *Offset;1011 }1012 1013 return sampleprof_error::success;1014}1015 1016std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles(1017 const DenseSet<StringRef> &FuncsToUse, SampleProfileMap &Profiles) {1018 const uint8_t *Start = Data;1019 1020 if (Remapper) {1021 for (auto Name : FuncsToUse) {1022 Remapper->insert(Name);1023 }1024 }1025 1026 if (ProfileIsCS) {1027 assert(useFuncOffsetList());1028 DenseSet<uint64_t> FuncGuidsToUse;1029 if (useMD5()) {1030 for (auto Name : FuncsToUse)1031 FuncGuidsToUse.insert(Function::getGUIDAssumingExternalLinkage(Name));1032 }1033 1034 // For each function in current module, load all context profiles for1035 // the function as well as their callee contexts which can help profile1036 // guided importing for ThinLTO. This can be achieved by walking1037 // through an ordered context container, where contexts are laid out1038 // as if they were walked in preorder of a context trie. While1039 // traversing the trie, a link to the highest common ancestor node is1040 // kept so that all of its decendants will be loaded.1041 const SampleContext *CommonContext = nullptr;1042 for (const auto &NameOffset : FuncOffsetList) {1043 const auto &FContext = NameOffset.first;1044 FunctionId FName = FContext.getFunction();1045 StringRef FNameString;1046 if (!useMD5())1047 FNameString = FName.stringRef();1048 1049 // For function in the current module, keep its farthest ancestor1050 // context. This can be used to load itself and its child and1051 // sibling contexts.1052 if ((useMD5() && FuncGuidsToUse.count(FName.getHashCode())) ||1053 (!useMD5() && (FuncsToUse.count(FNameString) ||1054 (Remapper && Remapper->exist(FNameString))))) {1055 if (!CommonContext || !CommonContext->isPrefixOf(FContext))1056 CommonContext = &FContext;1057 }1058 1059 if (CommonContext == &FContext ||1060 (CommonContext && CommonContext->isPrefixOf(FContext))) {1061 // Load profile for the current context which originated from1062 // the common ancestor.1063 const uint8_t *FuncProfileAddr = Start + NameOffset.second;1064 if (std::error_code EC = readFuncProfile(FuncProfileAddr))1065 return EC;1066 }1067 }1068 } else if (useMD5()) {1069 assert(!useFuncOffsetList());1070 for (auto Name : FuncsToUse) {1071 auto GUID = MD5Hash(Name);1072 auto iter = FuncOffsetTable.find(GUID);1073 if (iter == FuncOffsetTable.end())1074 continue;1075 const uint8_t *FuncProfileAddr = Start + iter->second;1076 if (std::error_code EC = readFuncProfile(FuncProfileAddr, Profiles))1077 return EC;1078 }1079 } else if (Remapper) {1080 assert(useFuncOffsetList());1081 for (auto NameOffset : FuncOffsetList) {1082 SampleContext FContext(NameOffset.first);1083 auto FuncName = FContext.getFunction();1084 StringRef FuncNameStr = FuncName.stringRef();1085 if (!FuncsToUse.count(FuncNameStr) && !Remapper->exist(FuncNameStr))1086 continue;1087 const uint8_t *FuncProfileAddr = Start + NameOffset.second;1088 if (std::error_code EC = readFuncProfile(FuncProfileAddr, Profiles))1089 return EC;1090 }1091 } else {1092 assert(!useFuncOffsetList());1093 for (auto Name : FuncsToUse) {1094 1095 auto iter = FuncOffsetTable.find(MD5Hash(Name));1096 if (iter == FuncOffsetTable.end())1097 continue;1098 const uint8_t *FuncProfileAddr = Start + iter->second;1099 if (std::error_code EC = readFuncProfile(FuncProfileAddr, Profiles))1100 return EC;1101 }1102 }1103 1104 return sampleprof_error::success;1105}1106 1107std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {1108 // Collect functions used by current module if the Reader has been1109 // given a module.1110 // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName1111 // which will query FunctionSamples::HasUniqSuffix, so it has to be1112 // called after FunctionSamples::HasUniqSuffix is set, i.e. after1113 // NameTable section is read.1114 bool LoadFuncsToBeUsed = collectFuncsFromModule();1115 1116 // When LoadFuncsToBeUsed is false, we are using LLVM tool, need to read all1117 // profiles.1118 if (!LoadFuncsToBeUsed) {1119 while (Data < End) {1120 if (std::error_code EC = readFuncProfile(Data))1121 return EC;1122 }1123 assert(Data == End && "More data is read than expected");1124 } else {1125 // Load function profiles on demand.1126 if (std::error_code EC = readFuncProfiles(FuncsToUse, Profiles))1127 return EC;1128 Data = End;1129 }1130 assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&1131 "Cannot have both context-sensitive and regular profile");1132 assert((!CSProfileCount || ProfileIsCS) &&1133 "Section flag should be consistent with actual profile");1134 return sampleprof_error::success;1135}1136 1137std::error_code SampleProfileReaderExtBinaryBase::readProfileSymbolList() {1138 if (!ProfSymList)1139 ProfSymList = std::make_unique<ProfileSymbolList>();1140 1141 if (std::error_code EC = ProfSymList->read(Data, End - Data))1142 return EC;1143 1144 Data = End;1145 return sampleprof_error::success;1146}1147 1148std::error_code SampleProfileReaderExtBinaryBase::decompressSection(1149 const uint8_t *SecStart, const uint64_t SecSize,1150 const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) {1151 Data = SecStart;1152 End = SecStart + SecSize;1153 auto DecompressSize = readNumber<uint64_t>();1154 if (std::error_code EC = DecompressSize.getError())1155 return EC;1156 DecompressBufSize = *DecompressSize;1157 1158 auto CompressSize = readNumber<uint64_t>();1159 if (std::error_code EC = CompressSize.getError())1160 return EC;1161 1162 if (!llvm::compression::zlib::isAvailable())1163 return sampleprof_error::zlib_unavailable;1164 1165 uint8_t *Buffer = Allocator.Allocate<uint8_t>(DecompressBufSize);1166 size_t UCSize = DecompressBufSize;1167 llvm::Error E = compression::zlib::decompress(ArrayRef(Data, *CompressSize),1168 Buffer, UCSize);1169 if (E)1170 return sampleprof_error::uncompress_failed;1171 DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer);1172 return sampleprof_error::success;1173}1174 1175std::error_code SampleProfileReaderExtBinaryBase::readImpl() {1176 const uint8_t *BufStart =1177 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());1178 1179 for (auto &Entry : SecHdrTable) {1180 // Skip empty section.1181 if (!Entry.Size)1182 continue;1183 1184 // Skip sections without inlined functions when SkipFlatProf is true.1185 if (SkipFlatProf && hasSecFlag(Entry, SecCommonFlags::SecFlagFlat))1186 continue;1187 1188 const uint8_t *SecStart = BufStart + Entry.Offset;1189 uint64_t SecSize = Entry.Size;1190 1191 // If the section is compressed, decompress it into a buffer1192 // DecompressBuf before reading the actual data. The pointee of1193 // 'Data' will be changed to buffer hold by DecompressBuf1194 // temporarily when reading the actual data.1195 bool isCompressed = hasSecFlag(Entry, SecCommonFlags::SecFlagCompress);1196 if (isCompressed) {1197 const uint8_t *DecompressBuf;1198 uint64_t DecompressBufSize;1199 if (std::error_code EC = decompressSection(1200 SecStart, SecSize, DecompressBuf, DecompressBufSize))1201 return EC;1202 SecStart = DecompressBuf;1203 SecSize = DecompressBufSize;1204 }1205 1206 if (std::error_code EC = readOneSection(SecStart, SecSize, Entry))1207 return EC;1208 if (Data != SecStart + SecSize)1209 return sampleprof_error::malformed;1210 1211 // Change the pointee of 'Data' from DecompressBuf to original Buffer.1212 if (isCompressed) {1213 Data = BufStart + Entry.Offset;1214 End = BufStart + Buffer->getBufferSize();1215 }1216 }1217 1218 return sampleprof_error::success;1219}1220 1221std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) {1222 if (Magic == SPMagic())1223 return sampleprof_error::success;1224 return sampleprof_error::bad_magic;1225}1226 1227std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) {1228 if (Magic == SPMagic(SPF_Ext_Binary))1229 return sampleprof_error::success;1230 return sampleprof_error::bad_magic;1231}1232 1233std::error_code SampleProfileReaderBinary::readNameTable() {1234 auto Size = readNumber<size_t>();1235 if (std::error_code EC = Size.getError())1236 return EC;1237 1238 // Normally if useMD5 is true, the name table should have MD5 values, not1239 // strings, however in the case that ExtBinary profile has multiple name1240 // tables mixing string and MD5, all of them have to be normalized to use MD5,1241 // because optimization passes can only handle either type.1242 bool UseMD5 = useMD5();1243 1244 NameTable.clear();1245 NameTable.reserve(*Size);1246 if (!ProfileIsCS) {1247 MD5SampleContextTable.clear();1248 if (UseMD5)1249 MD5SampleContextTable.reserve(*Size);1250 else1251 // If we are using strings, delay MD5 computation since only a portion of1252 // names are used by top level functions. Use 0 to indicate MD5 value is1253 // to be calculated as no known string has a MD5 value of 0.1254 MD5SampleContextTable.resize(*Size);1255 }1256 for (size_t I = 0; I < *Size; ++I) {1257 auto Name(readString());1258 if (std::error_code EC = Name.getError())1259 return EC;1260 if (UseMD5) {1261 FunctionId FID(*Name);1262 if (!ProfileIsCS)1263 MD5SampleContextTable.emplace_back(FID.getHashCode());1264 NameTable.emplace_back(FID);1265 } else1266 NameTable.push_back(FunctionId(*Name));1267 }1268 if (!ProfileIsCS)1269 MD5SampleContextStart = MD5SampleContextTable.data();1270 return sampleprof_error::success;1271}1272 1273std::error_code1274SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,1275 bool FixedLengthMD5) {1276 if (FixedLengthMD5) {1277 if (!IsMD5)1278 errs() << "If FixedLengthMD5 is true, UseMD5 has to be true";1279 auto Size = readNumber<size_t>();1280 if (std::error_code EC = Size.getError())1281 return EC;1282 1283 assert(Data + (*Size) * sizeof(uint64_t) == End &&1284 "Fixed length MD5 name table does not contain specified number of "1285 "entries");1286 if (Data + (*Size) * sizeof(uint64_t) > End)1287 return sampleprof_error::truncated;1288 1289 NameTable.clear();1290 NameTable.reserve(*Size);1291 for (size_t I = 0; I < *Size; ++I) {1292 using namespace support;1293 uint64_t FID = endian::read<uint64_t, unaligned>(1294 Data + I * sizeof(uint64_t), endianness::little);1295 NameTable.emplace_back(FunctionId(FID));1296 }1297 if (!ProfileIsCS)1298 MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);1299 Data = Data + (*Size) * sizeof(uint64_t);1300 return sampleprof_error::success;1301 }1302 1303 if (IsMD5) {1304 assert(!FixedLengthMD5 && "FixedLengthMD5 should be unreachable here");1305 auto Size = readNumber<size_t>();1306 if (std::error_code EC = Size.getError())1307 return EC;1308 1309 NameTable.clear();1310 NameTable.reserve(*Size);1311 if (!ProfileIsCS)1312 MD5SampleContextTable.resize(*Size);1313 for (size_t I = 0; I < *Size; ++I) {1314 auto FID = readNumber<uint64_t>();1315 if (std::error_code EC = FID.getError())1316 return EC;1317 if (!ProfileIsCS)1318 support::endian::write64le(&MD5SampleContextTable[I], *FID);1319 NameTable.emplace_back(FunctionId(*FID));1320 }1321 if (!ProfileIsCS)1322 MD5SampleContextStart = MD5SampleContextTable.data();1323 return sampleprof_error::success;1324 }1325 1326 return SampleProfileReaderBinary::readNameTable();1327}1328 1329// Read in the CS name table section, which basically contains a list of context1330// vectors. Each element of a context vector, aka a frame, refers to the1331// underlying raw function names that are stored in the name table, as well as1332// a callsite identifier that only makes sense for non-leaf frames.1333std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() {1334 auto Size = readNumber<size_t>();1335 if (std::error_code EC = Size.getError())1336 return EC;1337 1338 CSNameTable.clear();1339 CSNameTable.reserve(*Size);1340 if (ProfileIsCS) {1341 // Delay MD5 computation of CS context until they are needed. Use 0 to1342 // indicate MD5 value is to be calculated as no known string has a MD51343 // value of 0.1344 MD5SampleContextTable.clear();1345 MD5SampleContextTable.resize(*Size);1346 MD5SampleContextStart = MD5SampleContextTable.data();1347 }1348 for (size_t I = 0; I < *Size; ++I) {1349 CSNameTable.emplace_back(SampleContextFrameVector());1350 auto ContextSize = readNumber<uint32_t>();1351 if (std::error_code EC = ContextSize.getError())1352 return EC;1353 for (uint32_t J = 0; J < *ContextSize; ++J) {1354 auto FName(readStringFromTable());1355 if (std::error_code EC = FName.getError())1356 return EC;1357 auto LineOffset = readNumber<uint64_t>();1358 if (std::error_code EC = LineOffset.getError())1359 return EC;1360 1361 if (!isOffsetLegal(*LineOffset))1362 return sampleprof_error::illegal_line_offset;1363 1364 auto Discriminator = readNumber<uint64_t>();1365 if (std::error_code EC = Discriminator.getError())1366 return EC;1367 1368 CSNameTable.back().emplace_back(1369 FName.get(), LineLocation(LineOffset.get(), Discriminator.get()));1370 }1371 }1372 1373 return sampleprof_error::success;1374}1375 1376std::error_code1377SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,1378 FunctionSamples *FProfile) {1379 if (Data < End) {1380 if (ProfileIsProbeBased) {1381 auto Checksum = readNumber<uint64_t>();1382 if (std::error_code EC = Checksum.getError())1383 return EC;1384 if (FProfile)1385 FProfile->setFunctionHash(*Checksum);1386 }1387 1388 if (ProfileHasAttribute) {1389 auto Attributes = readNumber<uint32_t>();1390 if (std::error_code EC = Attributes.getError())1391 return EC;1392 if (FProfile)1393 FProfile->getContext().setAllAttributes(*Attributes);1394 }1395 1396 if (!ProfileIsCS) {1397 // Read all the attributes for inlined function calls.1398 auto NumCallsites = readNumber<uint32_t>();1399 if (std::error_code EC = NumCallsites.getError())1400 return EC;1401 1402 for (uint32_t J = 0; J < *NumCallsites; ++J) {1403 auto LineOffset = readNumber<uint64_t>();1404 if (std::error_code EC = LineOffset.getError())1405 return EC;1406 1407 auto Discriminator = readNumber<uint64_t>();1408 if (std::error_code EC = Discriminator.getError())1409 return EC;1410 1411 auto FContextHash(readSampleContextFromTable());1412 if (std::error_code EC = FContextHash.getError())1413 return EC;1414 1415 auto &[FContext, Hash] = *FContextHash;1416 FunctionSamples *CalleeProfile = nullptr;1417 if (FProfile) {1418 CalleeProfile = const_cast<FunctionSamples *>(1419 &FProfile->functionSamplesAt(LineLocation(1420 *LineOffset,1421 *Discriminator))[FContext.getFunction()]);1422 }1423 if (std::error_code EC =1424 readFuncMetadata(ProfileHasAttribute, CalleeProfile))1425 return EC;1426 }1427 }1428 }1429 1430 return sampleprof_error::success;1431}1432 1433std::error_code SampleProfileReaderExtBinaryBase::readFuncMetadata(1434 bool ProfileHasAttribute, DenseSet<FunctionSamples *> &Profiles) {1435 if (FuncMetadataIndex.empty())1436 return sampleprof_error::success;1437 1438 for (auto *FProfile : Profiles) {1439 auto R = FuncMetadataIndex.find(FProfile->getContext().getHashCode());1440 if (R == FuncMetadataIndex.end())1441 continue;1442 1443 Data = R->second.first;1444 End = R->second.second;1445 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))1446 return EC;1447 assert(Data == End && "More data is read than expected");1448 }1449 return sampleprof_error::success;1450}1451 1452std::error_code1453SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) {1454 while (Data < End) {1455 auto FContextHash(readSampleContextFromTable());1456 if (std::error_code EC = FContextHash.getError())1457 return EC;1458 auto &[FContext, Hash] = *FContextHash;1459 FunctionSamples *FProfile = nullptr;1460 auto It = Profiles.find(FContext);1461 if (It != Profiles.end())1462 FProfile = &It->second;1463 1464 const uint8_t *Start = Data;1465 if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile))1466 return EC;1467 1468 FuncMetadataIndex[FContext.getHashCode()] = {Start, Data};1469 }1470 1471 assert(Data == End && "More data is read than expected");1472 return sampleprof_error::success;1473}1474 1475std::error_code1476SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint64_t Idx) {1477 SecHdrTableEntry Entry;1478 auto Type = readUnencodedNumber<uint64_t>();1479 if (std::error_code EC = Type.getError())1480 return EC;1481 Entry.Type = static_cast<SecType>(*Type);1482 1483 auto Flags = readUnencodedNumber<uint64_t>();1484 if (std::error_code EC = Flags.getError())1485 return EC;1486 Entry.Flags = *Flags;1487 1488 auto Offset = readUnencodedNumber<uint64_t>();1489 if (std::error_code EC = Offset.getError())1490 return EC;1491 Entry.Offset = *Offset;1492 1493 auto Size = readUnencodedNumber<uint64_t>();1494 if (std::error_code EC = Size.getError())1495 return EC;1496 Entry.Size = *Size;1497 1498 Entry.LayoutIndex = Idx;1499 SecHdrTable.push_back(std::move(Entry));1500 return sampleprof_error::success;1501}1502 1503std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() {1504 auto EntryNum = readUnencodedNumber<uint64_t>();1505 if (std::error_code EC = EntryNum.getError())1506 return EC;1507 1508 for (uint64_t i = 0; i < (*EntryNum); i++)1509 if (std::error_code EC = readSecHdrTableEntry(i))1510 return EC;1511 1512 return sampleprof_error::success;1513}1514 1515std::error_code SampleProfileReaderExtBinaryBase::readHeader() {1516 const uint8_t *BufStart =1517 reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());1518 Data = BufStart;1519 End = BufStart + Buffer->getBufferSize();1520 1521 if (std::error_code EC = readMagicIdent())1522 return EC;1523 1524 if (std::error_code EC = readSecHdrTable())1525 return EC;1526 1527 return sampleprof_error::success;1528}1529 1530uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {1531 uint64_t Size = 0;1532 for (auto &Entry : SecHdrTable) {1533 if (Entry.Type == Type)1534 Size += Entry.Size;1535 }1536 return Size;1537}1538 1539uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {1540 // Sections in SecHdrTable is not necessarily in the same order as1541 // sections in the profile because section like FuncOffsetTable needs1542 // to be written after section LBRProfile but needs to be read before1543 // section LBRProfile, so we cannot simply use the last entry in1544 // SecHdrTable to calculate the file size.1545 uint64_t FileSize = 0;1546 for (auto &Entry : SecHdrTable) {1547 FileSize = std::max(Entry.Offset + Entry.Size, FileSize);1548 }1549 return FileSize;1550}1551 1552static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) {1553 std::string Flags;1554 if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress))1555 Flags.append("{compressed,");1556 else1557 Flags.append("{");1558 1559 if (hasSecFlag(Entry, SecCommonFlags::SecFlagFlat))1560 Flags.append("flat,");1561 1562 switch (Entry.Type) {1563 case SecNameTable:1564 if (hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5))1565 Flags.append("fixlenmd5,");1566 else if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name))1567 Flags.append("md5,");1568 if (hasSecFlag(Entry, SecNameTableFlags::SecFlagUniqSuffix))1569 Flags.append("uniq,");1570 break;1571 case SecProfSummary:1572 if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial))1573 Flags.append("partial,");1574 if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext))1575 Flags.append("context,");1576 if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsPreInlined))1577 Flags.append("preInlined,");1578 if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator))1579 Flags.append("fs-discriminator,");1580 break;1581 case SecFuncOffsetTable:1582 if (hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered))1583 Flags.append("ordered,");1584 break;1585 case SecFuncMetadata:1586 if (hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased))1587 Flags.append("probe,");1588 if (hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute))1589 Flags.append("attr,");1590 break;1591 default:1592 break;1593 }1594 char &last = Flags.back();1595 if (last == ',')1596 last = '}';1597 else1598 Flags.append("}");1599 return Flags;1600}1601 1602bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {1603 uint64_t TotalSecsSize = 0;1604 for (auto &Entry : SecHdrTable) {1605 OS << getSecName(Entry.Type) << " - Offset: " << Entry.Offset1606 << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry)1607 << "\n";1608 ;1609 TotalSecsSize += Entry.Size;1610 }1611 uint64_t HeaderSize = SecHdrTable.front().Offset;1612 assert(HeaderSize + TotalSecsSize == getFileSize() &&1613 "Size of 'header + sections' doesn't match the total size of profile");1614 1615 OS << "Header Size: " << HeaderSize << "\n";1616 OS << "Total Sections Size: " << TotalSecsSize << "\n";1617 OS << "File Size: " << getFileSize() << "\n";1618 return true;1619}1620 1621std::error_code SampleProfileReaderBinary::readMagicIdent() {1622 // Read and check the magic identifier.1623 auto Magic = readNumber<uint64_t>();1624 if (std::error_code EC = Magic.getError())1625 return EC;1626 else if (std::error_code EC = verifySPMagic(*Magic))1627 return EC;1628 1629 // Read the version number.1630 auto Version = readNumber<uint64_t>();1631 if (std::error_code EC = Version.getError())1632 return EC;1633 else if (*Version != SPVersion())1634 return sampleprof_error::unsupported_version;1635 1636 return sampleprof_error::success;1637}1638 1639std::error_code SampleProfileReaderBinary::readHeader() {1640 Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart());1641 End = Data + Buffer->getBufferSize();1642 1643 if (std::error_code EC = readMagicIdent())1644 return EC;1645 1646 if (std::error_code EC = readSummary())1647 return EC;1648 1649 if (std::error_code EC = readNameTable())1650 return EC;1651 return sampleprof_error::success;1652}1653 1654std::error_code SampleProfileReaderBinary::readSummaryEntry(1655 std::vector<ProfileSummaryEntry> &Entries) {1656 auto Cutoff = readNumber<uint64_t>();1657 if (std::error_code EC = Cutoff.getError())1658 return EC;1659 1660 auto MinBlockCount = readNumber<uint64_t>();1661 if (std::error_code EC = MinBlockCount.getError())1662 return EC;1663 1664 auto NumBlocks = readNumber<uint64_t>();1665 if (std::error_code EC = NumBlocks.getError())1666 return EC;1667 1668 Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks);1669 return sampleprof_error::success;1670}1671 1672std::error_code SampleProfileReaderBinary::readSummary() {1673 auto TotalCount = readNumber<uint64_t>();1674 if (std::error_code EC = TotalCount.getError())1675 return EC;1676 1677 auto MaxBlockCount = readNumber<uint64_t>();1678 if (std::error_code EC = MaxBlockCount.getError())1679 return EC;1680 1681 auto MaxFunctionCount = readNumber<uint64_t>();1682 if (std::error_code EC = MaxFunctionCount.getError())1683 return EC;1684 1685 auto NumBlocks = readNumber<uint64_t>();1686 if (std::error_code EC = NumBlocks.getError())1687 return EC;1688 1689 auto NumFunctions = readNumber<uint64_t>();1690 if (std::error_code EC = NumFunctions.getError())1691 return EC;1692 1693 auto NumSummaryEntries = readNumber<uint64_t>();1694 if (std::error_code EC = NumSummaryEntries.getError())1695 return EC;1696 1697 std::vector<ProfileSummaryEntry> Entries;1698 for (unsigned i = 0; i < *NumSummaryEntries; i++) {1699 std::error_code EC = readSummaryEntry(Entries);1700 if (EC != sampleprof_error::success)1701 return EC;1702 }1703 Summary = std::make_unique<ProfileSummary>(1704 ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0,1705 *MaxFunctionCount, *NumBlocks, *NumFunctions);1706 1707 return sampleprof_error::success;1708}1709 1710bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) {1711 const uint8_t *Data =1712 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());1713 uint64_t Magic = decodeULEB128(Data);1714 return Magic == SPMagic();1715}1716 1717bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) {1718 const uint8_t *Data =1719 reinterpret_cast<const uint8_t *>(Buffer.getBufferStart());1720 uint64_t Magic = decodeULEB128(Data);1721 return Magic == SPMagic(SPF_Ext_Binary);1722}1723 1724std::error_code SampleProfileReaderGCC::skipNextWord() {1725 uint32_t dummy;1726 if (!GcovBuffer.readInt(dummy))1727 return sampleprof_error::truncated;1728 return sampleprof_error::success;1729}1730 1731template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() {1732 if (sizeof(T) <= sizeof(uint32_t)) {1733 uint32_t Val;1734 if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max())1735 return static_cast<T>(Val);1736 } else if (sizeof(T) <= sizeof(uint64_t)) {1737 uint64_t Val;1738 if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max())1739 return static_cast<T>(Val);1740 }1741 1742 std::error_code EC = sampleprof_error::malformed;1743 reportError(0, EC.message());1744 return EC;1745}1746 1747ErrorOr<StringRef> SampleProfileReaderGCC::readString() {1748 StringRef Str;1749 if (!GcovBuffer.readString(Str))1750 return sampleprof_error::truncated;1751 return Str;1752}1753 1754std::error_code SampleProfileReaderGCC::readHeader() {1755 // Read the magic identifier.1756 if (!GcovBuffer.readGCDAFormat())1757 return sampleprof_error::unrecognized_format;1758 1759 // Read the version number. Note - the GCC reader does not validate this1760 // version, but the profile creator generates v704.1761 GCOV::GCOVVersion version;1762 if (!GcovBuffer.readGCOVVersion(version))1763 return sampleprof_error::unrecognized_format;1764 1765 if (version != GCOV::V407)1766 return sampleprof_error::unsupported_version;1767 1768 // Skip the empty integer.1769 if (std::error_code EC = skipNextWord())1770 return EC;1771 1772 return sampleprof_error::success;1773}1774 1775std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) {1776 uint32_t Tag;1777 if (!GcovBuffer.readInt(Tag))1778 return sampleprof_error::truncated;1779 1780 if (Tag != Expected)1781 return sampleprof_error::malformed;1782 1783 if (std::error_code EC = skipNextWord())1784 return EC;1785 1786 return sampleprof_error::success;1787}1788 1789std::error_code SampleProfileReaderGCC::readNameTable() {1790 if (std::error_code EC = readSectionTag(GCOVTagAFDOFileNames))1791 return EC;1792 1793 uint32_t Size;1794 if (!GcovBuffer.readInt(Size))1795 return sampleprof_error::truncated;1796 1797 for (uint32_t I = 0; I < Size; ++I) {1798 StringRef Str;1799 if (!GcovBuffer.readString(Str))1800 return sampleprof_error::truncated;1801 Names.push_back(std::string(Str));1802 }1803 1804 return sampleprof_error::success;1805}1806 1807std::error_code SampleProfileReaderGCC::readFunctionProfiles() {1808 if (std::error_code EC = readSectionTag(GCOVTagAFDOFunction))1809 return EC;1810 1811 uint32_t NumFunctions;1812 if (!GcovBuffer.readInt(NumFunctions))1813 return sampleprof_error::truncated;1814 1815 InlineCallStack Stack;1816 for (uint32_t I = 0; I < NumFunctions; ++I)1817 if (std::error_code EC = readOneFunctionProfile(Stack, true, 0))1818 return EC;1819 1820 computeSummary();1821 return sampleprof_error::success;1822}1823 1824std::error_code SampleProfileReaderGCC::readOneFunctionProfile(1825 const InlineCallStack &InlineStack, bool Update, uint32_t Offset) {1826 uint64_t HeadCount = 0;1827 if (InlineStack.size() == 0)1828 if (!GcovBuffer.readInt64(HeadCount))1829 return sampleprof_error::truncated;1830 1831 uint32_t NameIdx;1832 if (!GcovBuffer.readInt(NameIdx))1833 return sampleprof_error::truncated;1834 1835 StringRef Name(Names[NameIdx]);1836 1837 uint32_t NumPosCounts;1838 if (!GcovBuffer.readInt(NumPosCounts))1839 return sampleprof_error::truncated;1840 1841 uint32_t NumCallsites;1842 if (!GcovBuffer.readInt(NumCallsites))1843 return sampleprof_error::truncated;1844 1845 FunctionSamples *FProfile = nullptr;1846 if (InlineStack.size() == 0) {1847 // If this is a top function that we have already processed, do not1848 // update its profile again. This happens in the presence of1849 // function aliases. Since these aliases share the same function1850 // body, there will be identical replicated profiles for the1851 // original function. In this case, we simply not bother updating1852 // the profile of the original function.1853 FProfile = &Profiles[FunctionId(Name)];1854 FProfile->addHeadSamples(HeadCount);1855 if (FProfile->getTotalSamples() > 0)1856 Update = false;1857 } else {1858 // Otherwise, we are reading an inlined instance. The top of the1859 // inline stack contains the profile of the caller. Insert this1860 // callee in the caller's CallsiteMap.1861 FunctionSamples *CallerProfile = InlineStack.front();1862 uint32_t LineOffset = Offset >> 16;1863 uint32_t Discriminator = Offset & 0xffff;1864 FProfile = &CallerProfile->functionSamplesAt(1865 LineLocation(LineOffset, Discriminator))[FunctionId(Name)];1866 }1867 FProfile->setFunction(FunctionId(Name));1868 1869 for (uint32_t I = 0; I < NumPosCounts; ++I) {1870 uint32_t Offset;1871 if (!GcovBuffer.readInt(Offset))1872 return sampleprof_error::truncated;1873 1874 uint32_t NumTargets;1875 if (!GcovBuffer.readInt(NumTargets))1876 return sampleprof_error::truncated;1877 1878 uint64_t Count;1879 if (!GcovBuffer.readInt64(Count))1880 return sampleprof_error::truncated;1881 1882 // The line location is encoded in the offset as:1883 // high 16 bits: line offset to the start of the function.1884 // low 16 bits: discriminator.1885 uint32_t LineOffset = Offset >> 16;1886 uint32_t Discriminator = Offset & 0xffff;1887 1888 InlineCallStack NewStack;1889 NewStack.push_back(FProfile);1890 llvm::append_range(NewStack, InlineStack);1891 if (Update) {1892 // Walk up the inline stack, adding the samples on this line to1893 // the total sample count of the callers in the chain.1894 for (auto *CallerProfile : NewStack)1895 CallerProfile->addTotalSamples(Count);1896 1897 // Update the body samples for the current profile.1898 FProfile->addBodySamples(LineOffset, Discriminator, Count);1899 }1900 1901 // Process the list of functions called at an indirect call site.1902 // These are all the targets that a function pointer (or virtual1903 // function) resolved at runtime.1904 for (uint32_t J = 0; J < NumTargets; J++) {1905 uint32_t HistVal;1906 if (!GcovBuffer.readInt(HistVal))1907 return sampleprof_error::truncated;1908 1909 if (HistVal != HIST_TYPE_INDIR_CALL_TOPN)1910 return sampleprof_error::malformed;1911 1912 uint64_t TargetIdx;1913 if (!GcovBuffer.readInt64(TargetIdx))1914 return sampleprof_error::truncated;1915 StringRef TargetName(Names[TargetIdx]);1916 1917 uint64_t TargetCount;1918 if (!GcovBuffer.readInt64(TargetCount))1919 return sampleprof_error::truncated;1920 1921 if (Update)1922 FProfile->addCalledTargetSamples(LineOffset, Discriminator,1923 FunctionId(TargetName),1924 TargetCount);1925 }1926 }1927 1928 // Process all the inlined callers into the current function. These1929 // are all the callsites that were inlined into this function.1930 for (uint32_t I = 0; I < NumCallsites; I++) {1931 // The offset is encoded as:1932 // high 16 bits: line offset to the start of the function.1933 // low 16 bits: discriminator.1934 uint32_t Offset;1935 if (!GcovBuffer.readInt(Offset))1936 return sampleprof_error::truncated;1937 InlineCallStack NewStack;1938 NewStack.push_back(FProfile);1939 llvm::append_range(NewStack, InlineStack);1940 if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset))1941 return EC;1942 }1943 1944 return sampleprof_error::success;1945}1946 1947/// Read a GCC AutoFDO profile.1948///1949/// This format is generated by the Linux Perf conversion tool at1950/// https://github.com/google/autofdo.1951std::error_code SampleProfileReaderGCC::readImpl() {1952 assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator");1953 // Read the string table.1954 if (std::error_code EC = readNameTable())1955 return EC;1956 1957 // Read the source profile.1958 if (std::error_code EC = readFunctionProfiles())1959 return EC;1960 1961 return sampleprof_error::success;1962}1963 1964bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) {1965 StringRef Magic(Buffer.getBufferStart());1966 return Magic == "adcg*704";1967}1968 1969void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {1970 // If the reader uses MD5 to represent string, we can't remap it because1971 // we don't know what the original function names were.1972 if (Reader.useMD5()) {1973 Ctx.diagnose(DiagnosticInfoSampleProfile(1974 Reader.getBuffer()->getBufferIdentifier(),1975 "Profile data remapping cannot be applied to profile data "1976 "using MD5 names (original mangled names are not available).",1977 DS_Warning));1978 return;1979 }1980 1981 // CSSPGO-TODO: Remapper is not yet supported.1982 // We will need to remap the entire context string.1983 assert(Remappings && "should be initialized while creating remapper");1984 for (auto &Sample : Reader.getProfiles()) {1985 DenseSet<FunctionId> NamesInSample;1986 Sample.second.findAllNames(NamesInSample);1987 for (auto &Name : NamesInSample) {1988 StringRef NameStr = Name.stringRef();1989 if (auto Key = Remappings->insert(NameStr))1990 NameMap.insert({Key, NameStr});1991 }1992 }1993 1994 RemappingApplied = true;1995}1996 1997std::optional<StringRef>1998SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) {1999 if (auto Key = Remappings->lookup(Fname)) {2000 StringRef Result = NameMap.lookup(Key);2001 if (!Result.empty())2002 return Result;2003 }2004 return std::nullopt;2005}2006 2007/// Prepare a memory buffer for the contents of \p Filename.2008///2009/// \returns an error code indicating the status of the buffer.2010static ErrorOr<std::unique_ptr<MemoryBuffer>>2011setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {2012 auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()2013 : FS.getBufferForFile(Filename);2014 if (std::error_code EC = BufferOrErr.getError())2015 return EC;2016 auto Buffer = std::move(BufferOrErr.get());2017 2018 return std::move(Buffer);2019}2020 2021/// Create a sample profile reader based on the format of the input file.2022///2023/// \param Filename The file to open.2024///2025/// \param C The LLVM context to use to emit diagnostics.2026///2027/// \param P The FSDiscriminatorPass.2028///2029/// \param RemapFilename The file used for profile remapping.2030///2031/// \returns an error code indicating the status of the created reader.2032ErrorOr<std::unique_ptr<SampleProfileReader>>2033SampleProfileReader::create(StringRef Filename, LLVMContext &C,2034 vfs::FileSystem &FS, FSDiscriminatorPass P,2035 StringRef RemapFilename) {2036 auto BufferOrError = setupMemoryBuffer(Filename, FS);2037 if (std::error_code EC = BufferOrError.getError())2038 return EC;2039 return create(BufferOrError.get(), C, FS, P, RemapFilename);2040}2041 2042/// Create a sample profile remapper from the given input, to remap the2043/// function names in the given profile data.2044///2045/// \param Filename The file to open.2046///2047/// \param Reader The profile reader the remapper is going to be applied to.2048///2049/// \param C The LLVM context to use to emit diagnostics.2050///2051/// \returns an error code indicating the status of the created reader.2052ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>2053SampleProfileReaderItaniumRemapper::create(StringRef Filename,2054 vfs::FileSystem &FS,2055 SampleProfileReader &Reader,2056 LLVMContext &C) {2057 auto BufferOrError = setupMemoryBuffer(Filename, FS);2058 if (std::error_code EC = BufferOrError.getError())2059 return EC;2060 return create(BufferOrError.get(), Reader, C);2061}2062 2063/// Create a sample profile remapper from the given input, to remap the2064/// function names in the given profile data.2065///2066/// \param B The memory buffer to create the reader from (assumes ownership).2067///2068/// \param C The LLVM context to use to emit diagnostics.2069///2070/// \param Reader The profile reader the remapper is going to be applied to.2071///2072/// \returns an error code indicating the status of the created reader.2073ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>>2074SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B,2075 SampleProfileReader &Reader,2076 LLVMContext &C) {2077 auto Remappings = std::make_unique<SymbolRemappingReader>();2078 if (Error E = Remappings->read(*B)) {2079 handleAllErrors(2080 std::move(E), [&](const SymbolRemappingParseError &ParseError) {2081 C.diagnose(DiagnosticInfoSampleProfile(B->getBufferIdentifier(),2082 ParseError.getLineNum(),2083 ParseError.getMessage()));2084 });2085 return sampleprof_error::malformed;2086 }2087 2088 return std::make_unique<SampleProfileReaderItaniumRemapper>(2089 std::move(B), std::move(Remappings), Reader);2090}2091 2092/// Create a sample profile reader based on the format of the input data.2093///2094/// \param B The memory buffer to create the reader from (assumes ownership).2095///2096/// \param C The LLVM context to use to emit diagnostics.2097///2098/// \param P The FSDiscriminatorPass.2099///2100/// \param RemapFilename The file used for profile remapping.2101///2102/// \returns an error code indicating the status of the created reader.2103ErrorOr<std::unique_ptr<SampleProfileReader>>2104SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C,2105 vfs::FileSystem &FS, FSDiscriminatorPass P,2106 StringRef RemapFilename) {2107 std::unique_ptr<SampleProfileReader> Reader;2108 if (SampleProfileReaderRawBinary::hasFormat(*B))2109 Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C));2110 else if (SampleProfileReaderExtBinary::hasFormat(*B))2111 Reader.reset(new SampleProfileReaderExtBinary(std::move(B), C));2112 else if (SampleProfileReaderGCC::hasFormat(*B))2113 Reader.reset(new SampleProfileReaderGCC(std::move(B), C));2114 else if (SampleProfileReaderText::hasFormat(*B))2115 Reader.reset(new SampleProfileReaderText(std::move(B), C));2116 else2117 return sampleprof_error::unrecognized_format;2118 2119 if (!RemapFilename.empty()) {2120 auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create(2121 RemapFilename, FS, *Reader, C);2122 if (std::error_code EC = ReaderOrErr.getError()) {2123 std::string Msg = "Could not create remapper: " + EC.message();2124 C.diagnose(DiagnosticInfoSampleProfile(RemapFilename, Msg));2125 return EC;2126 }2127 Reader->Remapper = std::move(ReaderOrErr.get());2128 }2129 2130 if (std::error_code EC = Reader->readHeader()) {2131 return EC;2132 }2133 2134 Reader->setDiscriminatorMaskedBitFrom(P);2135 2136 return std::move(Reader);2137}2138 2139// For text and GCC file formats, we compute the summary after reading the2140// profile. Binary format has the profile summary in its header.2141void SampleProfileReader::computeSummary() {2142 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);2143 Summary = Builder.computeSummaryForProfiles(Profiles);2144}2145