992 lines · cpp
1//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the class that writes LLVM sample profiles. It10// supports two file formats: text and binary. The textual representation11// is useful for debugging and testing purposes. The binary representation12// is more compact, resulting in smaller file sizes. However, they can13// both be used interchangeably.14//15// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the16// supported formats.17//18//===----------------------------------------------------------------------===//19 20#include "llvm/ProfileData/SampleProfWriter.h"21#include "llvm/ADT/StringRef.h"22#include "llvm/ProfileData/ProfileCommon.h"23#include "llvm/ProfileData/SampleProf.h"24#include "llvm/Support/Compression.h"25#include "llvm/Support/EndianStream.h"26#include "llvm/Support/ErrorOr.h"27#include "llvm/Support/FileSystem.h"28#include "llvm/Support/LEB128.h"29#include "llvm/Support/MD5.h"30#include "llvm/Support/raw_ostream.h"31#include <cmath>32#include <cstdint>33#include <memory>34#include <set>35#include <system_error>36#include <utility>37#include <vector>38 39#define DEBUG_TYPE "llvm-profdata"40 41using namespace llvm;42using namespace sampleprof;43 44// To begin with, make this option off by default.45static cl::opt<bool> ExtBinaryWriteVTableTypeProf(46 "extbinary-write-vtable-type-prof", cl::init(false), cl::Hidden,47 cl::desc("Write vtable type profile in ext-binary sample profile writer"));48 49namespace llvm {50namespace support {51namespace endian {52namespace {53 54// Adapter class to llvm::support::endian::Writer for pwrite().55struct SeekableWriter {56 raw_pwrite_stream &OS;57 endianness Endian;58 SeekableWriter(raw_pwrite_stream &OS, endianness Endian)59 : OS(OS), Endian(Endian) {}60 61 template <typename ValueType>62 void pwrite(ValueType Val, size_t Offset) {63 std::string StringBuf;64 raw_string_ostream SStream(StringBuf);65 Writer(SStream, Endian).write(Val);66 OS.pwrite(StringBuf.data(), StringBuf.size(), Offset);67 }68};69 70} // namespace71} // namespace endian72} // namespace support73} // namespace llvm74 75DefaultFunctionPruningStrategy::DefaultFunctionPruningStrategy(76 SampleProfileMap &ProfileMap, size_t OutputSizeLimit)77 : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {78 sortFuncProfiles(ProfileMap, SortedFunctions);79}80 81void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {82 double D = (double)OutputSizeLimit / CurrentOutputSize;83 size_t NewSize = (size_t)round(ProfileMap.size() * D * D);84 size_t NumToRemove = ProfileMap.size() - NewSize;85 if (NumToRemove < 1)86 NumToRemove = 1;87 88 assert(NumToRemove <= SortedFunctions.size());89 for (const NameFunctionSamples &E :90 llvm::drop_begin(SortedFunctions, SortedFunctions.size() - NumToRemove))91 ProfileMap.erase(E.first);92 SortedFunctions.resize(SortedFunctions.size() - NumToRemove);93}94 95std::error_code SampleProfileWriter::writeWithSizeLimitInternal(96 SampleProfileMap &ProfileMap, size_t OutputSizeLimit,97 FunctionPruningStrategy *Strategy) {98 if (OutputSizeLimit == 0)99 return write(ProfileMap);100 101 size_t OriginalFunctionCount = ProfileMap.size();102 103 std::unique_ptr<raw_ostream> OriginalOutputStream;104 OutputStream.swap(OriginalOutputStream);105 106 size_t IterationCount = 0;107 size_t TotalSize;108 109 SmallVector<char> StringBuffer;110 do {111 StringBuffer.clear();112 OutputStream.reset(new raw_svector_ostream(StringBuffer));113 if (std::error_code EC = write(ProfileMap))114 return EC;115 116 TotalSize = StringBuffer.size();117 // On Windows every "\n" is actually written as "\r\n" to disk but not to118 // memory buffer, this difference should be added when considering the total119 // output size.120#ifdef _WIN32121 if (Format == SPF_Text)122 TotalSize += LineCount;123#endif124 if (TotalSize <= OutputSizeLimit)125 break;126 127 Strategy->Erase(TotalSize);128 IterationCount++;129 } while (ProfileMap.size() != 0);130 131 if (ProfileMap.size() == 0)132 return sampleprof_error::too_large;133 134 OutputStream.swap(OriginalOutputStream);135 OutputStream->write(StringBuffer.data(), StringBuffer.size());136 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount137 << " functions, reduced to " << ProfileMap.size() << " in "138 << IterationCount << " iterations\n");139 // Silence warning on Release build.140 (void)OriginalFunctionCount;141 (void)IterationCount;142 return sampleprof_error::success;143}144 145std::error_code146SampleProfileWriter::writeFuncProfiles(const SampleProfileMap &ProfileMap) {147 std::vector<NameFunctionSamples> V;148 sortFuncProfiles(ProfileMap, V);149 for (const auto &I : V) {150 if (std::error_code EC = writeSample(*I.second))151 return EC;152 }153 return sampleprof_error::success;154}155 156std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {157 if (std::error_code EC = writeHeader(ProfileMap))158 return EC;159 160 if (std::error_code EC = writeFuncProfiles(ProfileMap))161 return EC;162 163 return sampleprof_error::success;164}165 166/// Return the current position and prepare to use it as the start167/// position of a section given the section type \p Type and its position168/// \p LayoutIdx in SectionHdrLayout.169uint64_t170SampleProfileWriterExtBinaryBase::markSectionStart(SecType Type,171 uint32_t LayoutIdx) {172 uint64_t SectionStart = OutputStream->tell();173 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");174 const auto &Entry = SectionHdrLayout[LayoutIdx];175 assert(Entry.Type == Type && "Unexpected section type");176 // Use LocalBuf as a temporary output for writting data.177 if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress))178 LocalBufStream.swap(OutputStream);179 return SectionStart;180}181 182std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {183 if (!llvm::compression::zlib::isAvailable())184 return sampleprof_error::zlib_unavailable;185 std::string &UncompressedStrings =186 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();187 if (UncompressedStrings.size() == 0)188 return sampleprof_error::success;189 auto &OS = *OutputStream;190 SmallVector<uint8_t, 128> CompressedStrings;191 compression::zlib::compress(arrayRefFromStringRef(UncompressedStrings),192 CompressedStrings,193 compression::zlib::BestSizeCompression);194 encodeULEB128(UncompressedStrings.size(), OS);195 encodeULEB128(CompressedStrings.size(), OS);196 OS << toStringRef(CompressedStrings);197 UncompressedStrings.clear();198 return sampleprof_error::success;199}200 201/// Add a new section into section header table given the section type202/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the203/// location \p SectionStart where the section should be written to.204std::error_code SampleProfileWriterExtBinaryBase::addNewSection(205 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {206 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");207 const auto &Entry = SectionHdrLayout[LayoutIdx];208 assert(Entry.Type == Type && "Unexpected section type");209 if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress)) {210 LocalBufStream.swap(OutputStream);211 if (std::error_code EC = compressAndOutput())212 return EC;213 }214 SecHdrTable.push_back({Type, Entry.Flags, SectionStart - FileStart,215 OutputStream->tell() - SectionStart, LayoutIdx});216 return sampleprof_error::success;217}218 219std::error_code220SampleProfileWriterExtBinaryBase::write(const SampleProfileMap &ProfileMap) {221 // When calling write on a different profile map, existing states should be222 // cleared.223 NameTable.clear();224 CSNameTable.clear();225 SecHdrTable.clear();226 227 if (std::error_code EC = writeHeader(ProfileMap))228 return EC;229 230 std::string LocalBuf;231 LocalBufStream = std::make_unique<raw_string_ostream>(LocalBuf);232 if (std::error_code EC = writeSections(ProfileMap))233 return EC;234 235 if (std::error_code EC = writeSecHdrTable())236 return EC;237 238 return sampleprof_error::success;239}240 241std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(242 const SampleContext &Context) {243 if (Context.hasContext())244 return writeCSNameIdx(Context);245 else246 return SampleProfileWriterBinary::writeNameIdx(Context.getFunction());247}248 249std::error_code250SampleProfileWriterExtBinaryBase::writeCSNameIdx(const SampleContext &Context) {251 const auto &Ret = CSNameTable.find(Context);252 if (Ret == CSNameTable.end())253 return sampleprof_error::truncated_name_table;254 encodeULEB128(Ret->second, *OutputStream);255 return sampleprof_error::success;256}257 258std::error_code259SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {260 uint64_t Offset = OutputStream->tell();261 auto &Context = S.getContext();262 FuncOffsetTable[Context] = Offset - SecLBRProfileStart;263 encodeULEB128(S.getHeadSamples(), *OutputStream);264 return writeBody(S);265}266 267std::error_code SampleProfileWriterExtBinaryBase::writeFuncOffsetTable() {268 auto &OS = *OutputStream;269 270 // Write out the table size.271 encodeULEB128(FuncOffsetTable.size(), OS);272 273 // Write out FuncOffsetTable.274 auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {275 if (std::error_code EC = writeContextIdx(Context))276 return EC;277 encodeULEB128(Offset, OS);278 return (std::error_code)sampleprof_error::success;279 };280 281 if (FunctionSamples::ProfileIsCS) {282 // Sort the contexts before writing them out. This is to help fast load all283 // context profiles for a function as well as their callee contexts which284 // can help profile-guided importing for ThinLTO.285 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(286 FuncOffsetTable.begin(), FuncOffsetTable.end());287 for (const auto &Entry : OrderedFuncOffsetTable) {288 if (std::error_code EC = WriteItem(Entry.first, Entry.second))289 return EC;290 }291 addSectionFlag(SecFuncOffsetTable, SecFuncOffsetFlags::SecFlagOrdered);292 } else {293 for (const auto &Entry : FuncOffsetTable) {294 if (std::error_code EC = WriteItem(Entry.first, Entry.second))295 return EC;296 }297 }298 299 FuncOffsetTable.clear();300 return sampleprof_error::success;301}302 303std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(304 const FunctionSamples &FunctionProfile) {305 auto &OS = *OutputStream;306 if (std::error_code EC = writeContextIdx(FunctionProfile.getContext()))307 return EC;308 309 if (FunctionSamples::ProfileIsProbeBased)310 encodeULEB128(FunctionProfile.getFunctionHash(), OS);311 if (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined) {312 encodeULEB128(FunctionProfile.getContext().getAllAttributes(), OS);313 }314 315 if (!FunctionSamples::ProfileIsCS) {316 // Recursively emit attributes for all callee samples.317 uint64_t NumCallsites = 0;318 for (const auto &J : FunctionProfile.getCallsiteSamples())319 NumCallsites += J.second.size();320 encodeULEB128(NumCallsites, OS);321 for (const auto &J : FunctionProfile.getCallsiteSamples()) {322 for (const auto &FS : J.second) {323 LineLocation Loc = J.first;324 encodeULEB128(Loc.LineOffset, OS);325 encodeULEB128(Loc.Discriminator, OS);326 if (std::error_code EC = writeFuncMetadata(FS.second))327 return EC;328 }329 }330 }331 332 return sampleprof_error::success;333}334 335std::error_code SampleProfileWriterExtBinaryBase::writeFuncMetadata(336 const SampleProfileMap &Profiles) {337 if (!FunctionSamples::ProfileIsProbeBased && !FunctionSamples::ProfileIsCS &&338 !FunctionSamples::ProfileIsPreInlined)339 return sampleprof_error::success;340 for (const auto &Entry : Profiles) {341 if (std::error_code EC = writeFuncMetadata(Entry.second))342 return EC;343 }344 return sampleprof_error::success;345}346 347std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {348 if (!UseMD5)349 return SampleProfileWriterBinary::writeNameTable();350 351 auto &OS = *OutputStream;352 std::set<FunctionId> V;353 stablizeNameTable(NameTable, V);354 355 // Write out the MD5 name table. We wrote unencoded MD5 so reader can356 // retrieve the name using the name index without having to read the357 // whole name table.358 encodeULEB128(NameTable.size(), OS);359 support::endian::Writer Writer(OS, llvm::endianness::little);360 for (auto N : V)361 Writer.write(N.getHashCode());362 return sampleprof_error::success;363}364 365std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(366 const SampleProfileMap &ProfileMap) {367 for (const auto &I : ProfileMap) {368 addContext(I.second.getContext());369 addNames(I.second);370 }371 372 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag373 // so compiler won't strip the suffix during profile matching after374 // seeing the flag in the profile.375 // Original names are unavailable if using MD5, so this option has no use.376 if (!UseMD5) {377 for (const auto &I : NameTable) {378 if (I.first.stringRef().contains(FunctionSamples::UniqSuffix)) {379 addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix);380 break;381 }382 }383 }384 385 if (auto EC = writeNameTable())386 return EC;387 return sampleprof_error::success;388}389 390std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {391 // Sort the names to make CSNameTable deterministic.392 std::set<SampleContext> OrderedContexts;393 for (const auto &I : CSNameTable)394 OrderedContexts.insert(I.first);395 assert(OrderedContexts.size() == CSNameTable.size() &&396 "Unmatched ordered and unordered contexts");397 uint64_t I = 0;398 for (auto &Context : OrderedContexts)399 CSNameTable[Context] = I++;400 401 auto &OS = *OutputStream;402 encodeULEB128(OrderedContexts.size(), OS);403 support::endian::Writer Writer(OS, llvm::endianness::little);404 for (auto Context : OrderedContexts) {405 auto Frames = Context.getContextFrames();406 encodeULEB128(Frames.size(), OS);407 for (auto &Callsite : Frames) {408 if (std::error_code EC = writeNameIdx(Callsite.Func))409 return EC;410 encodeULEB128(Callsite.Location.LineOffset, OS);411 encodeULEB128(Callsite.Location.Discriminator, OS);412 }413 }414 415 return sampleprof_error::success;416}417 418std::error_code419SampleProfileWriterExtBinaryBase::writeProfileSymbolListSection() {420 if (ProfSymList && ProfSymList->size() > 0)421 if (std::error_code EC = ProfSymList->write(*OutputStream))422 return EC;423 424 return sampleprof_error::success;425}426 427std::error_code SampleProfileWriterExtBinaryBase::writeOneSection(428 SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) {429 // The setting of SecFlagCompress should happen before markSectionStart.430 if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())431 setToCompressSection(SecProfileSymbolList);432 if (Type == SecFuncMetadata && FunctionSamples::ProfileIsProbeBased)433 addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagIsProbeBased);434 if (Type == SecFuncMetadata &&435 (FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsPreInlined))436 addSectionFlag(SecFuncMetadata, SecFuncMetadataFlags::SecFlagHasAttribute);437 if (Type == SecProfSummary && FunctionSamples::ProfileIsCS)438 addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFullContext);439 if (Type == SecProfSummary && FunctionSamples::ProfileIsPreInlined)440 addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagIsPreInlined);441 if (Type == SecProfSummary && FunctionSamples::ProfileIsFS)442 addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFSDiscriminator);443 if (Type == SecProfSummary && ExtBinaryWriteVTableTypeProf)444 addSectionFlag(SecProfSummary,445 SecProfSummaryFlags::SecFlagHasVTableTypeProf);446 447 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);448 switch (Type) {449 case SecProfSummary:450 computeSummary(ProfileMap);451 if (auto EC = writeSummary())452 return EC;453 break;454 case SecNameTable:455 if (auto EC = writeNameTableSection(ProfileMap))456 return EC;457 break;458 case SecCSNameTable:459 if (auto EC = writeCSNameTableSection())460 return EC;461 break;462 case SecLBRProfile:463 SecLBRProfileStart = OutputStream->tell();464 if (std::error_code EC = writeFuncProfiles(ProfileMap))465 return EC;466 break;467 case SecFuncOffsetTable:468 if (auto EC = writeFuncOffsetTable())469 return EC;470 break;471 case SecFuncMetadata:472 if (std::error_code EC = writeFuncMetadata(ProfileMap))473 return EC;474 break;475 case SecProfileSymbolList:476 if (auto EC = writeProfileSymbolListSection())477 return EC;478 break;479 default:480 if (auto EC = writeCustomSection(Type))481 return EC;482 break;483 }484 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))485 return EC;486 return sampleprof_error::success;487}488 489SampleProfileWriterExtBinary::SampleProfileWriterExtBinary(490 std::unique_ptr<raw_ostream> &OS)491 : SampleProfileWriterExtBinaryBase(OS) {492 WriteVTableProf = ExtBinaryWriteVTableTypeProf;493}494 495std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(496 const SampleProfileMap &ProfileMap) {497 // The const indices passed to writeOneSection below are specifying the498 // positions of the sections in SectionHdrLayout. Look at499 // initSectionHdrLayout to find out where each section is located in500 // SectionHdrLayout.501 if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))502 return EC;503 if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))504 return EC;505 if (auto EC = writeOneSection(SecCSNameTable, 2, ProfileMap))506 return EC;507 if (auto EC = writeOneSection(SecLBRProfile, 4, ProfileMap))508 return EC;509 if (auto EC = writeOneSection(SecProfileSymbolList, 5, ProfileMap))510 return EC;511 if (auto EC = writeOneSection(SecFuncOffsetTable, 3, ProfileMap))512 return EC;513 if (auto EC = writeOneSection(SecFuncMetadata, 6, ProfileMap))514 return EC;515 return sampleprof_error::success;516}517 518static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,519 SampleProfileMap &ContextProfileMap,520 SampleProfileMap &NoContextProfileMap) {521 for (const auto &I : ProfileMap) {522 if (I.second.getCallsiteSamples().size())523 ContextProfileMap.insert({I.first, I.second});524 else525 NoContextProfileMap.insert({I.first, I.second});526 }527}528 529std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(530 const SampleProfileMap &ProfileMap) {531 SampleProfileMap ContextProfileMap, NoContextProfileMap;532 splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap);533 534 if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))535 return EC;536 if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))537 return EC;538 if (auto EC = writeOneSection(SecLBRProfile, 3, ContextProfileMap))539 return EC;540 if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ContextProfileMap))541 return EC;542 // Mark the section to have no context. Note section flag needs to be set543 // before writing the section.544 addSectionFlag(5, SecCommonFlags::SecFlagFlat);545 if (auto EC = writeOneSection(SecLBRProfile, 5, NoContextProfileMap))546 return EC;547 // Mark the section to have no context. Note section flag needs to be set548 // before writing the section.549 addSectionFlag(4, SecCommonFlags::SecFlagFlat);550 if (auto EC = writeOneSection(SecFuncOffsetTable, 4, NoContextProfileMap))551 return EC;552 if (auto EC = writeOneSection(SecProfileSymbolList, 6, ProfileMap))553 return EC;554 if (auto EC = writeOneSection(SecFuncMetadata, 7, ProfileMap))555 return EC;556 557 return sampleprof_error::success;558}559 560std::error_code SampleProfileWriterExtBinary::writeSections(561 const SampleProfileMap &ProfileMap) {562 std::error_code EC;563 if (SecLayout == DefaultLayout)564 EC = writeDefaultLayout(ProfileMap);565 else if (SecLayout == CtxSplitLayout)566 EC = writeCtxSplitLayout(ProfileMap);567 else568 llvm_unreachable("Unsupported layout");569 return EC;570}571 572/// Write samples to a text file.573///574/// Note: it may be tempting to implement this in terms of575/// FunctionSamples::print(). Please don't. The dump functionality is intended576/// for debugging and has no specified form.577///578/// The format used here is more structured and deliberate because579/// it needs to be parsed by the SampleProfileReaderText class.580std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {581 auto &OS = *OutputStream;582 if (FunctionSamples::ProfileIsCS)583 OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();584 else585 OS << S.getFunction() << ":" << S.getTotalSamples();586 587 if (Indent == 0)588 OS << ":" << S.getHeadSamples();589 OS << "\n";590 LineCount++;591 592 SampleSorter<LineLocation, SampleRecord> SortedSamples(S.getBodySamples());593 for (const auto &I : SortedSamples.get()) {594 LineLocation Loc = I->first;595 const SampleRecord &Sample = I->second;596 OS.indent(Indent + 1);597 Loc.print(OS);598 OS << ": " << Sample.getSamples();599 600 for (const auto &J : Sample.getSortedCallTargets())601 OS << " " << J.first << ":" << J.second;602 OS << "\n";603 LineCount++;604 605 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);606 Map && !Map->empty()) {607 OS.indent(Indent + 1);608 Loc.print(OS);609 OS << ": ";610 OS << kVTableProfPrefix;611 for (const auto [TypeName, Count] : *Map) {612 OS << TypeName << ":" << Count << " ";613 }614 OS << "\n";615 LineCount++;616 }617 }618 619 SampleSorter<LineLocation, FunctionSamplesMap> SortedCallsiteSamples(620 S.getCallsiteSamples());621 Indent += 1;622 for (const auto *Element : SortedCallsiteSamples.get()) {623 // Element is a pointer to a pair of LineLocation and FunctionSamplesMap.624 const auto &[Loc, FunctionSamplesMap] = *Element;625 for (const FunctionSamples &CalleeSamples :626 make_second_range(FunctionSamplesMap)) {627 OS.indent(Indent);628 Loc.print(OS);629 OS << ": ";630 if (std::error_code EC = writeSample(CalleeSamples))631 return EC;632 }633 634 if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc);635 Map && !Map->empty()) {636 OS.indent(Indent);637 Loc.print(OS);638 OS << ": ";639 OS << kVTableProfPrefix;640 for (const auto [TypeId, Count] : *Map) {641 OS << TypeId << ":" << Count << " ";642 }643 OS << "\n";644 LineCount++;645 }646 }647 648 Indent -= 1;649 650 if (FunctionSamples::ProfileIsProbeBased) {651 OS.indent(Indent + 1);652 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";653 LineCount++;654 }655 656 if (S.getContext().getAllAttributes()) {657 OS.indent(Indent + 1);658 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";659 LineCount++;660 }661 662 if (Indent == 0 && MarkFlatProfiles && S.getCallsiteSamples().size() == 0)663 OS << " !Flat\n";664 665 return sampleprof_error::success;666}667 668std::error_code669SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {670 assert(!Context.hasContext() && "cs profile is not supported");671 return writeNameIdx(Context.getFunction());672}673 674std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {675 auto &NTable = getNameTable();676 const auto &Ret = NTable.find(FName);677 if (Ret == NTable.end())678 return sampleprof_error::truncated_name_table;679 encodeULEB128(Ret->second, *OutputStream);680 return sampleprof_error::success;681}682 683void SampleProfileWriterBinary::addName(FunctionId FName) {684 auto &NTable = getNameTable();685 NTable.insert(std::make_pair(FName, 0));686}687 688void SampleProfileWriterBinary::addContext(const SampleContext &Context) {689 addName(Context.getFunction());690}691 692void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {693 // Add all the names in indirect call targets.694 for (const auto &I : S.getBodySamples()) {695 const SampleRecord &Sample = I.second;696 for (const auto &J : Sample.getCallTargets())697 addName(J.first);698 }699 700 // Recursively add all the names for inlined callsites.701 for (const auto &J : S.getCallsiteSamples())702 for (const auto &FS : J.second) {703 const FunctionSamples &CalleeSamples = FS.second;704 addName(CalleeSamples.getFunction());705 addNames(CalleeSamples);706 }707 708 if (!WriteVTableProf)709 return;710 // Add all the vtable names to NameTable.711 for (const auto &VTableAccessCountMap :712 llvm::make_second_range(S.getCallsiteTypeCounts())) {713 // Add type name to NameTable.714 for (const auto Type : llvm::make_first_range(VTableAccessCountMap)) {715 addName(Type);716 }717 }718}719 720void SampleProfileWriterExtBinaryBase::addContext(721 const SampleContext &Context) {722 if (Context.hasContext()) {723 for (auto &Callsite : Context.getContextFrames())724 SampleProfileWriterBinary::addName(Callsite.Func);725 CSNameTable.insert(std::make_pair(Context, 0));726 } else {727 SampleProfileWriterBinary::addName(Context.getFunction());728 }729}730 731void SampleProfileWriterBinary::stablizeNameTable(732 MapVector<FunctionId, uint32_t> &NameTable, std::set<FunctionId> &V) {733 // Sort the names to make NameTable deterministic.734 for (const auto &I : NameTable)735 V.insert(I.first);736 int i = 0;737 for (const FunctionId &N : V)738 NameTable[N] = i++;739}740 741std::error_code SampleProfileWriterBinary::writeNameTable() {742 auto &OS = *OutputStream;743 std::set<FunctionId> V;744 stablizeNameTable(NameTable, V);745 746 // Write out the name table.747 encodeULEB128(NameTable.size(), OS);748 for (auto N : V) {749 OS << N;750 encodeULEB128(0, OS);751 }752 return sampleprof_error::success;753}754 755std::error_code756SampleProfileWriterBinary::writeMagicIdent(SampleProfileFormat Format) {757 auto &OS = *OutputStream;758 // Write file magic identifier.759 encodeULEB128(SPMagic(Format), OS);760 encodeULEB128(SPVersion(), OS);761 return sampleprof_error::success;762}763 764std::error_code765SampleProfileWriterBinary::writeHeader(const SampleProfileMap &ProfileMap) {766 // When calling write on a different profile map, existing names should be767 // cleared.768 NameTable.clear();769 770 writeMagicIdent(Format);771 772 computeSummary(ProfileMap);773 if (auto EC = writeSummary())774 return EC;775 776 // Generate the name table for all the functions referenced in the profile.777 for (const auto &I : ProfileMap) {778 addContext(I.second.getContext());779 addNames(I.second);780 }781 782 writeNameTable();783 return sampleprof_error::success;784}785 786void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {787 for (auto &Entry : SectionHdrLayout)788 addSecFlag(Entry, SecCommonFlags::SecFlagCompress);789}790 791void SampleProfileWriterExtBinaryBase::setToCompressSection(SecType Type) {792 addSectionFlag(Type, SecCommonFlags::SecFlagCompress);793}794 795void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {796 support::endian::Writer Writer(*OutputStream, llvm::endianness::little);797 798 Writer.write(static_cast<uint64_t>(SectionHdrLayout.size()));799 SecHdrTableOffset = OutputStream->tell();800 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {801 Writer.write(static_cast<uint64_t>(-1));802 Writer.write(static_cast<uint64_t>(-1));803 Writer.write(static_cast<uint64_t>(-1));804 Writer.write(static_cast<uint64_t>(-1));805 }806}807 808std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {809 assert(SecHdrTable.size() == SectionHdrLayout.size() &&810 "SecHdrTable entries doesn't match SectionHdrLayout");811 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);812 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {813 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;814 }815 816 // Write the section header table in the order specified in817 // SectionHdrLayout. SectionHdrLayout specifies the sections818 // order in which profile reader expect to read, so the section819 // header table should be written in the order in SectionHdrLayout.820 // Note that the section order in SecHdrTable may be different821 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable822 // needs to be computed after SecLBRProfile (the order in SecHdrTable),823 // but it needs to be read before SecLBRProfile (the order in824 // SectionHdrLayout). So we use IndexMap above to switch the order.825 support::endian::SeekableWriter Writer(826 static_cast<raw_pwrite_stream &>(*OutputStream),827 llvm::endianness::little);828 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();829 LayoutIdx++) {830 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&831 "Incorrect LayoutIdx in SecHdrTable");832 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];833 Writer.pwrite(static_cast<uint64_t>(Entry.Type),834 SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));835 Writer.pwrite(static_cast<uint64_t>(Entry.Flags),836 SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));837 Writer.pwrite(static_cast<uint64_t>(Entry.Offset),838 SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));839 Writer.pwrite(static_cast<uint64_t>(Entry.Size),840 SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));841 }842 843 return sampleprof_error::success;844}845 846std::error_code SampleProfileWriterExtBinaryBase::writeHeader(847 const SampleProfileMap &ProfileMap) {848 auto &OS = *OutputStream;849 FileStart = OS.tell();850 writeMagicIdent(Format);851 852 allocSecHdrTable();853 return sampleprof_error::success;854}855 856std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf(857 const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) {858 assert(WriteVTableProf &&859 "writeCallsiteVTableProf should not be called if WriteVTableProf is "860 "false");861 862 encodeULEB128(CallsiteTypeMap.size(), OS);863 for (const auto &[Loc, TypeMap] : CallsiteTypeMap) {864 Loc.serialize(OS);865 if (std::error_code EC = serializeTypeMap(TypeMap, getNameTable(), OS))866 return EC;867 }868 869 return sampleprof_error::success;870}871 872std::error_code SampleProfileWriterBinary::writeSummary() {873 auto &OS = *OutputStream;874 encodeULEB128(Summary->getTotalCount(), OS);875 encodeULEB128(Summary->getMaxCount(), OS);876 encodeULEB128(Summary->getMaxFunctionCount(), OS);877 encodeULEB128(Summary->getNumCounts(), OS);878 encodeULEB128(Summary->getNumFunctions(), OS);879 ArrayRef<ProfileSummaryEntry> Entries = Summary->getDetailedSummary();880 encodeULEB128(Entries.size(), OS);881 for (auto Entry : Entries) {882 encodeULEB128(Entry.Cutoff, OS);883 encodeULEB128(Entry.MinCount, OS);884 encodeULEB128(Entry.NumCounts, OS);885 }886 return sampleprof_error::success;887}888std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {889 auto &OS = *OutputStream;890 if (std::error_code EC = writeContextIdx(S.getContext()))891 return EC;892 893 encodeULEB128(S.getTotalSamples(), OS);894 895 // Emit all the body samples.896 encodeULEB128(S.getBodySamples().size(), OS);897 for (const auto &I : S.getBodySamples()) {898 LineLocation Loc = I.first;899 const SampleRecord &Sample = I.second;900 Loc.serialize(OS);901 Sample.serialize(OS, getNameTable());902 }903 904 // Recursively emit all the callsite samples.905 uint64_t NumCallsites = 0;906 for (const auto &J : S.getCallsiteSamples())907 NumCallsites += J.second.size();908 encodeULEB128(NumCallsites, OS);909 for (const auto &J : S.getCallsiteSamples())910 for (const auto &FS : J.second) {911 J.first.serialize(OS);912 if (std::error_code EC = writeBody(FS.second))913 return EC;914 }915 916 if (WriteVTableProf)917 return writeCallsiteVTableProf(S.getCallsiteTypeCounts(), OS);918 919 return sampleprof_error::success;920}921 922/// Write samples of a top-level function to a binary file.923///924/// \returns true if the samples were written successfully, false otherwise.925std::error_code926SampleProfileWriterBinary::writeSample(const FunctionSamples &S) {927 encodeULEB128(S.getHeadSamples(), *OutputStream);928 return writeBody(S);929}930 931/// Create a sample profile file writer based on the specified format.932///933/// \param Filename The file to create.934///935/// \param Format Encoding format for the profile file.936///937/// \returns an error code indicating the status of the created writer.938ErrorOr<std::unique_ptr<SampleProfileWriter>>939SampleProfileWriter::create(StringRef Filename, SampleProfileFormat Format) {940 std::error_code EC;941 std::unique_ptr<raw_ostream> OS;942 if (Format == SPF_Binary || Format == SPF_Ext_Binary)943 OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));944 else945 OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));946 if (EC)947 return EC;948 949 return create(OS, Format);950}951 952/// Create a sample profile stream writer based on the specified format.953///954/// \param OS The output stream to store the profile data to.955///956/// \param Format Encoding format for the profile file.957///958/// \returns an error code indicating the status of the created writer.959ErrorOr<std::unique_ptr<SampleProfileWriter>>960SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,961 SampleProfileFormat Format) {962 std::error_code EC;963 std::unique_ptr<SampleProfileWriter> Writer;964 965 // Currently only Text and Extended Binary format are supported for CSSPGO.966 if ((FunctionSamples::ProfileIsCS || FunctionSamples::ProfileIsProbeBased) &&967 Format == SPF_Binary)968 return sampleprof_error::unsupported_writing_format;969 970 if (Format == SPF_Binary)971 Writer.reset(new SampleProfileWriterRawBinary(OS));972 else if (Format == SPF_Ext_Binary)973 Writer.reset(new SampleProfileWriterExtBinary(OS));974 else if (Format == SPF_Text)975 Writer.reset(new SampleProfileWriterText(OS));976 else if (Format == SPF_GCC)977 EC = sampleprof_error::unsupported_writing_format;978 else979 EC = sampleprof_error::unrecognized_format;980 981 if (EC)982 return EC;983 984 Writer->Format = Format;985 return std::move(Writer);986}987 988void SampleProfileWriter::computeSummary(const SampleProfileMap &ProfileMap) {989 SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);990 Summary = Builder.computeSummaryForProfiles(ProfileMap);991}992