831 lines · cpp
1//===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains support for writing profiling data for clang's10// instrumentation based PGO and coverage.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/ProfileData/InstrProfWriter.h"15#include "llvm/ADT/STLExtras.h"16#include "llvm/ADT/StringRef.h"17#include "llvm/IR/ProfileSummary.h"18#include "llvm/ProfileData/DataAccessProf.h"19#include "llvm/ProfileData/IndexedMemProfData.h"20#include "llvm/ProfileData/InstrProf.h"21#include "llvm/ProfileData/ProfileCommon.h"22#include "llvm/Support/Compression.h"23#include "llvm/Support/EndianStream.h"24#include "llvm/Support/Error.h"25#include "llvm/Support/MemoryBuffer.h"26#include "llvm/Support/OnDiskHashTable.h"27#include "llvm/Support/raw_ostream.h"28#include <cstdint>29#include <ctime>30#include <memory>31#include <string>32#include <tuple>33#include <utility>34#include <vector>35 36using namespace llvm;37 38namespace llvm {39 40class InstrProfRecordWriterTrait {41public:42 using key_type = StringRef;43 using key_type_ref = StringRef;44 45 using data_type = const InstrProfWriter::ProfilingData *const;46 using data_type_ref = const InstrProfWriter::ProfilingData *const;47 48 using hash_value_type = uint64_t;49 using offset_type = uint64_t;50 51 llvm::endianness ValueProfDataEndianness = llvm::endianness::little;52 InstrProfSummaryBuilder *SummaryBuilder;53 InstrProfSummaryBuilder *CSSummaryBuilder;54 55 InstrProfRecordWriterTrait() = default;56 57 static hash_value_type ComputeHash(key_type_ref K) {58 return IndexedInstrProf::ComputeHash(K);59 }60 61 static std::pair<offset_type, offset_type>62 EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V) {63 using namespace support;64 65 endian::Writer LE(Out, llvm::endianness::little);66 67 offset_type N = K.size();68 LE.write<offset_type>(N);69 70 offset_type M = 0;71 for (const auto &ProfileData : *V) {72 const InstrProfRecord &ProfRecord = ProfileData.second;73 M += sizeof(uint64_t); // The function hash74 M += sizeof(uint64_t); // The size of the Counts vector75 M += ProfRecord.Counts.size() * sizeof(uint64_t);76 M += sizeof(uint64_t); // The size of the Bitmap vector77 M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t);78 79 // Value data80 M += ValueProfData::getSize(ProfileData.second);81 }82 LE.write<offset_type>(M);83 84 return std::make_pair(N, M);85 }86 87 void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N) {88 Out.write(K.data(), N);89 }90 91 void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type) {92 using namespace support;93 94 endian::Writer LE(Out, llvm::endianness::little);95 for (const auto &ProfileData : *V) {96 const InstrProfRecord &ProfRecord = ProfileData.second;97 if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))98 CSSummaryBuilder->addRecord(ProfRecord);99 else100 SummaryBuilder->addRecord(ProfRecord);101 102 LE.write<uint64_t>(ProfileData.first); // Function hash103 LE.write<uint64_t>(ProfRecord.Counts.size());104 for (uint64_t I : ProfRecord.Counts)105 LE.write<uint64_t>(I);106 107 LE.write<uint64_t>(ProfRecord.BitmapBytes.size());108 for (uint64_t I : ProfRecord.BitmapBytes)109 LE.write<uint64_t>(I);110 111 // Write value data112 std::unique_ptr<ValueProfData> VDataPtr =113 ValueProfData::serializeFrom(ProfileData.second);114 uint32_t S = VDataPtr->getSize();115 VDataPtr->swapBytesFromHost(ValueProfDataEndianness);116 Out.write((const char *)VDataPtr.get(), S);117 }118 }119};120 121} // end namespace llvm122 123InstrProfWriter::InstrProfWriter(124 bool Sparse, uint64_t TemporalProfTraceReservoirSize,125 uint64_t MaxTemporalProfTraceLength, bool WritePrevVersion,126 memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,127 bool MemprofGenerateRandomHotness,128 unsigned MemprofGenerateRandomHotnessSeed)129 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),130 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),131 InfoObj(new InstrProfRecordWriterTrait()),132 WritePrevVersion(WritePrevVersion),133 MemProfVersionRequested(MemProfVersionRequested),134 MemProfFullSchema(MemProfFullSchema),135 MemprofGenerateRandomHotness(MemprofGenerateRandomHotness) {136 // Set up the random number seed if requested.137 if (MemprofGenerateRandomHotness) {138 unsigned seed = MemprofGenerateRandomHotnessSeed139 ? MemprofGenerateRandomHotnessSeed140 : std::time(nullptr);141 errs() << "random hotness seed = " << seed << "\n";142 std::srand(seed);143 }144}145 146InstrProfWriter::~InstrProfWriter() { delete InfoObj; }147 148// Internal interface for testing purpose only.149void InstrProfWriter::setValueProfDataEndianness(llvm::endianness Endianness) {150 InfoObj->ValueProfDataEndianness = Endianness;151}152 153void InstrProfWriter::setOutputSparse(bool Sparse) { this->Sparse = Sparse; }154 155void InstrProfWriter::addRecord(NamedInstrProfRecord &&I, uint64_t Weight,156 function_ref<void(Error)> Warn) {157 auto Name = I.Name;158 auto Hash = I.Hash;159 addRecord(Name, Hash, std::move(I), Weight, Warn);160}161 162void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,163 OverlapStats &Overlap,164 OverlapStats &FuncLevelOverlap,165 const OverlapFuncFilters &FuncFilter) {166 auto Name = Other.Name;167 auto Hash = Other.Hash;168 Other.accumulateCounts(FuncLevelOverlap.Test);169 auto It = FunctionData.find(Name);170 if (It == FunctionData.end()) {171 Overlap.addOneUnique(FuncLevelOverlap.Test);172 return;173 }174 if (FuncLevelOverlap.Test.CountSum < 1.0f) {175 Overlap.Overlap.NumEntries += 1;176 return;177 }178 auto &ProfileDataMap = It->second;179 auto [Where, NewFunc] = ProfileDataMap.try_emplace(Hash);180 if (NewFunc) {181 Overlap.addOneMismatch(FuncLevelOverlap.Test);182 return;183 }184 InstrProfRecord &Dest = Where->second;185 186 uint64_t ValueCutoff = FuncFilter.ValueCutoff;187 if (!FuncFilter.NameFilter.empty() && Name.contains(FuncFilter.NameFilter))188 ValueCutoff = 0;189 190 Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);191}192 193void InstrProfWriter::addRecord(StringRef Name, uint64_t Hash,194 InstrProfRecord &&I, uint64_t Weight,195 function_ref<void(Error)> Warn) {196 auto &ProfileDataMap = FunctionData[Name];197 198 auto [Where, NewFunc] = ProfileDataMap.try_emplace(Hash);199 InstrProfRecord &Dest = Where->second;200 201 auto MapWarn = [&](instrprof_error E) {202 Warn(make_error<InstrProfError>(E));203 };204 205 if (NewFunc) {206 // We've never seen a function with this name and hash, add it.207 Dest = std::move(I);208 if (Weight > 1)209 Dest.scale(Weight, 1, MapWarn);210 } else {211 // We're updating a function we've seen before.212 Dest.merge(I, Weight, MapWarn);213 }214 215 Dest.sortValueData();216}217 218void InstrProfWriter::addMemProfRecord(219 const Function::GUID Id, const memprof::IndexedMemProfRecord &Record) {220 auto NewRecord = Record;221 // Provoke random hotness values if requested. We specify the lifetime access222 // density and lifetime length that will result in a cold or not cold hotness.223 // See the logic in getAllocType() in Analysis/MemoryProfileInfo.cpp.224 if (MemprofGenerateRandomHotness) {225 for (auto &Alloc : NewRecord.AllocSites) {226 // To get a not cold context, set the lifetime access density to the227 // maximum value and the lifetime to 0.228 uint64_t NewTLAD = std::numeric_limits<uint64_t>::max();229 uint64_t NewTL = 0;230 bool IsCold = std::rand() % 2;231 if (IsCold) {232 // To get a cold context, set the lifetime access density to 0 and the233 // lifetime to the maximum value.234 NewTLAD = 0;235 NewTL = std::numeric_limits<uint64_t>::max();236 }237 Alloc.Info.setTotalLifetimeAccessDensity(NewTLAD);238 Alloc.Info.setTotalLifetime(NewTL);239 }240 }241 MemProfSumBuilder.addRecord(NewRecord);242 auto [Iter, Inserted] = MemProfData.Records.insert({Id, NewRecord});243 // If we inserted a new record then we are done.244 if (Inserted) {245 return;246 }247 memprof::IndexedMemProfRecord &Existing = Iter->second;248 Existing.merge(NewRecord);249}250 251bool InstrProfWriter::addMemProfFrame(const memprof::FrameId Id,252 const memprof::Frame &Frame,253 function_ref<void(Error)> Warn) {254 auto [Iter, Inserted] = MemProfData.Frames.insert({Id, Frame});255 // If a mapping already exists for the current frame id and it does not256 // match the new mapping provided then reset the existing contents and bail257 // out. We don't support the merging of memprof data whose Frame -> Id258 // mapping across profiles is inconsistent.259 if (!Inserted && Iter->second != Frame) {260 Warn(make_error<InstrProfError>(instrprof_error::malformed,261 "frame to id mapping mismatch"));262 return false;263 }264 return true;265}266 267bool InstrProfWriter::addMemProfCallStack(268 const memprof::CallStackId CSId,269 const llvm::SmallVector<memprof::FrameId> &CallStack,270 function_ref<void(Error)> Warn) {271 auto [Iter, Inserted] = MemProfData.CallStacks.insert({CSId, CallStack});272 // If a mapping already exists for the current call stack id and it does not273 // match the new mapping provided then reset the existing contents and bail274 // out. We don't support the merging of memprof data whose CallStack -> Id275 // mapping across profiles is inconsistent.276 if (!Inserted && Iter->second != CallStack) {277 Warn(make_error<InstrProfError>(instrprof_error::malformed,278 "call stack to id mapping mismatch"));279 return false;280 }281 return true;282}283 284bool InstrProfWriter::addMemProfData(memprof::IndexedMemProfData Incoming,285 function_ref<void(Error)> Warn) {286 // Return immediately if everything is empty.287 if (Incoming.Frames.empty() && Incoming.CallStacks.empty() &&288 Incoming.Records.empty())289 return true;290 291 // Otherwise, every component must be non-empty.292 assert(!Incoming.Frames.empty() && !Incoming.CallStacks.empty() &&293 !Incoming.Records.empty());294 295 if (MemProfData.Frames.empty())296 MemProfData.Frames = std::move(Incoming.Frames);297 else298 for (const auto &[Id, F] : Incoming.Frames)299 if (addMemProfFrame(Id, F, Warn))300 return false;301 302 if (MemProfData.CallStacks.empty())303 MemProfData.CallStacks = std::move(Incoming.CallStacks);304 else305 for (const auto &[CSId, CS] : Incoming.CallStacks)306 if (addMemProfCallStack(CSId, CS, Warn))307 return false;308 309 // Add one record at a time if randomization is requested.310 if (MemProfData.Records.empty() && !MemprofGenerateRandomHotness) {311 // Need to manually add each record to the builder, which is otherwise done312 // in addMemProfRecord.313 for (const auto &[GUID, Record] : Incoming.Records)314 MemProfSumBuilder.addRecord(Record);315 MemProfData.Records = std::move(Incoming.Records);316 } else {317 for (const auto &[GUID, Record] : Incoming.Records)318 addMemProfRecord(GUID, Record);319 }320 321 return true;322}323 324void InstrProfWriter::addBinaryIds(ArrayRef<llvm::object::BuildID> BIs) {325 llvm::append_range(BinaryIds, BIs);326}327 328void InstrProfWriter::addDataAccessProfData(329 std::unique_ptr<memprof::DataAccessProfData> DataAccessProfDataIn) {330 DataAccessProfileData = std::move(DataAccessProfDataIn);331}332 333void InstrProfWriter::addTemporalProfileTraces(334 SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {335 if (TemporalProfTraces.size() > TemporalProfTraceReservoirSize)336 TemporalProfTraces.truncate(TemporalProfTraceReservoirSize);337 for (auto &Trace : SrcTraces)338 if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)339 Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);340 llvm::erase_if(SrcTraces, [](auto &T) { return T.FunctionNameRefs.empty(); });341 // If there are no source traces, it is probably because342 // --temporal-profile-max-trace-length=0 was set to deliberately remove all343 // traces. In that case, we do not want to increase the stream size344 if (SrcTraces.empty())345 return;346 // Add traces until our reservoir is full or we run out of source traces347 auto SrcTraceIt = SrcTraces.begin();348 while (TemporalProfTraces.size() < TemporalProfTraceReservoirSize &&349 SrcTraceIt < SrcTraces.end())350 TemporalProfTraces.push_back(*SrcTraceIt++);351 // Our reservoir is full, we need to sample the source stream352 llvm::shuffle(SrcTraceIt, SrcTraces.end(), RNG);353 for (uint64_t I = TemporalProfTraces.size();354 I < SrcStreamSize && SrcTraceIt < SrcTraces.end(); I++) {355 std::uniform_int_distribution<uint64_t> Distribution(0, I);356 uint64_t RandomIndex = Distribution(RNG);357 if (RandomIndex < TemporalProfTraces.size())358 TemporalProfTraces[RandomIndex] = *SrcTraceIt++;359 }360 TemporalProfTraceStreamSize += SrcStreamSize;361}362 363void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,364 function_ref<void(Error)> Warn) {365 for (auto &I : IPW.FunctionData)366 for (auto &Func : I.getValue())367 addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn);368 369 BinaryIds.reserve(BinaryIds.size() + IPW.BinaryIds.size());370 for (auto &I : IPW.BinaryIds)371 addBinaryIds(I);372 373 addTemporalProfileTraces(IPW.TemporalProfTraces,374 IPW.TemporalProfTraceStreamSize);375 376 MemProfData.Frames.reserve(IPW.MemProfData.Frames.size());377 for (auto &[FrameId, Frame] : IPW.MemProfData.Frames) {378 // If we weren't able to add the frame mappings then it doesn't make sense379 // to try to merge the records from this profile.380 if (!addMemProfFrame(FrameId, Frame, Warn))381 return;382 }383 384 MemProfData.CallStacks.reserve(IPW.MemProfData.CallStacks.size());385 for (auto &[CSId, CallStack] : IPW.MemProfData.CallStacks) {386 if (!addMemProfCallStack(CSId, CallStack, Warn))387 return;388 }389 390 MemProfData.Records.reserve(IPW.MemProfData.Records.size());391 for (auto &[GUID, Record] : IPW.MemProfData.Records) {392 addMemProfRecord(GUID, Record);393 }394}395 396bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {397 if (!Sparse)398 return true;399 for (const auto &Func : PD) {400 const InstrProfRecord &IPR = Func.second;401 if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; }))402 return true;403 if (llvm::any_of(IPR.BitmapBytes, [](uint8_t Byte) { return Byte > 0; }))404 return true;405 }406 return false;407}408 409static void setSummary(IndexedInstrProf::Summary *TheSummary,410 ProfileSummary &PS) {411 using namespace IndexedInstrProf;412 413 const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();414 TheSummary->NumSummaryFields = Summary::NumKinds;415 TheSummary->NumCutoffEntries = Res.size();416 TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());417 TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());418 TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());419 TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());420 TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());421 TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());422 for (unsigned I = 0; I < Res.size(); I++)423 TheSummary->setEntry(I, Res[I]);424}425 426uint64_t InstrProfWriter::writeHeader(const IndexedInstrProf::Header &Header,427 const bool WritePrevVersion,428 ProfOStream &OS) {429 // Only write out the first four fields.430 for (int I = 0; I < 4; I++)431 OS.write(reinterpret_cast<const uint64_t *>(&Header)[I]);432 433 // Remember the offset of the remaining fields to allow back patching later.434 auto BackPatchStartOffset = OS.tell();435 436 // Reserve the space for back patching later.437 OS.write(0); // HashOffset438 OS.write(0); // MemProfOffset439 OS.write(0); // BinaryIdOffset440 OS.write(0); // TemporalProfTracesOffset441 if (!WritePrevVersion)442 OS.write(0); // VTableNamesOffset443 444 return BackPatchStartOffset;445}446 447Error InstrProfWriter::writeBinaryIds(ProfOStream &OS) {448 // BinaryIdSection has two parts:449 // 1. uint64_t BinaryIdsSectionSize450 // 2. list of binary ids that consist of:451 // a. uint64_t BinaryIdLength452 // b. uint8_t BinaryIdData453 // c. uint8_t Padding (if necessary)454 // Calculate size of binary section.455 uint64_t BinaryIdsSectionSize = 0;456 457 // Remove duplicate binary ids.458 llvm::sort(BinaryIds);459 BinaryIds.erase(llvm::unique(BinaryIds), BinaryIds.end());460 461 for (const auto &BI : BinaryIds) {462 // Increment by binary id length data type size.463 BinaryIdsSectionSize += sizeof(uint64_t);464 // Increment by binary id data length, aligned to 8 bytes.465 BinaryIdsSectionSize += alignToPowerOf2(BI.size(), sizeof(uint64_t));466 }467 // Write binary ids section size.468 OS.write(BinaryIdsSectionSize);469 470 for (const auto &BI : BinaryIds) {471 uint64_t BILen = BI.size();472 // Write binary id length.473 OS.write(BILen);474 // Write binary id data.475 for (unsigned K = 0; K < BILen; K++)476 OS.writeByte(BI[K]);477 // Write padding if necessary.478 uint64_t PaddingSize = alignToPowerOf2(BILen, sizeof(uint64_t)) - BILen;479 for (unsigned K = 0; K < PaddingSize; K++)480 OS.writeByte(0);481 }482 483 return Error::success();484}485 486Error InstrProfWriter::writeVTableNames(ProfOStream &OS) {487 std::vector<std::string> VTableNameStrs;488 for (StringRef VTableName : VTableNames.keys())489 VTableNameStrs.push_back(VTableName.str());490 491 std::string CompressedVTableNames;492 if (!VTableNameStrs.empty())493 if (Error E = collectGlobalObjectNameStrings(494 VTableNameStrs, compression::zlib::isAvailable(),495 CompressedVTableNames))496 return E;497 498 const uint64_t CompressedStringLen = CompressedVTableNames.length();499 500 // Record the length of compressed string.501 OS.write(CompressedStringLen);502 503 // Write the chars in compressed strings.504 for (auto &c : CompressedVTableNames)505 OS.writeByte(static_cast<uint8_t>(c));506 507 // Pad up to a multiple of 8.508 // InstrProfReader could read bytes according to 'CompressedStringLen'.509 const uint64_t PaddedLength = alignTo(CompressedStringLen, 8);510 511 for (uint64_t K = CompressedStringLen; K < PaddedLength; K++)512 OS.writeByte(0);513 514 return Error::success();515}516 517Error InstrProfWriter::writeImpl(ProfOStream &OS) {518 using namespace IndexedInstrProf;519 using namespace support;520 521 OnDiskChainedHashTableGenerator<InstrProfRecordWriterTrait> Generator;522 523 InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);524 InfoObj->SummaryBuilder = &ISB;525 InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs);526 InfoObj->CSSummaryBuilder = &CSISB;527 528 // Populate the hash table generator.529 SmallVector<std::pair<StringRef, const ProfilingData *>> OrderedData;530 for (const auto &I : FunctionData)531 if (shouldEncodeData(I.getValue()))532 OrderedData.emplace_back((I.getKey()), &I.getValue());533 llvm::sort(OrderedData, less_first());534 for (const auto &I : OrderedData)535 Generator.insert(I.first, I.second);536 537 // Write the header.538 IndexedInstrProf::Header Header;539 Header.Version = WritePrevVersion540 ? IndexedInstrProf::ProfVersion::Version11541 : IndexedInstrProf::ProfVersion::CurrentVersion;542 // The WritePrevVersion handling will either need to be removed or updated543 // if the version is advanced beyond 12.544 static_assert(IndexedInstrProf::ProfVersion::CurrentVersion ==545 IndexedInstrProf::ProfVersion::Version13);546 if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))547 Header.Version |= VARIANT_MASK_IR_PROF;548 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))549 Header.Version |= VARIANT_MASK_CSIR_PROF;550 if (static_cast<bool>(ProfileKind &551 InstrProfKind::FunctionEntryInstrumentation))552 Header.Version |= VARIANT_MASK_INSTR_ENTRY;553 if (static_cast<bool>(ProfileKind &554 InstrProfKind::LoopEntriesInstrumentation))555 Header.Version |= VARIANT_MASK_INSTR_LOOP_ENTRIES;556 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))557 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;558 if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))559 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;560 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))561 Header.Version |= VARIANT_MASK_MEMPROF;562 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))563 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;564 565 const uint64_t BackPatchStartOffset =566 writeHeader(Header, WritePrevVersion, OS);567 568 // Reserve space to write profile summary data.569 uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size();570 uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);571 // Remember the summary offset.572 uint64_t SummaryOffset = OS.tell();573 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)574 OS.write(0);575 uint64_t CSSummaryOffset = 0;576 uint64_t CSSummarySize = 0;577 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {578 CSSummaryOffset = OS.tell();579 CSSummarySize = SummarySize / sizeof(uint64_t);580 for (unsigned I = 0; I < CSSummarySize; I++)581 OS.write(0);582 }583 584 // Write the hash table.585 uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);586 587 // Write the MemProf profile data if we have it.588 uint64_t MemProfSectionStart = 0;589 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {590 MemProfSectionStart = OS.tell();591 592 if (auto E = writeMemProf(593 OS, MemProfData, MemProfVersionRequested, MemProfFullSchema,594 std::move(DataAccessProfileData), MemProfSumBuilder.getSummary()))595 return E;596 }597 598 uint64_t BinaryIdSectionStart = OS.tell();599 if (auto E = writeBinaryIds(OS))600 return E;601 602 uint64_t VTableNamesSectionStart = OS.tell();603 604 if (!WritePrevVersion)605 if (Error E = writeVTableNames(OS))606 return E;607 608 uint64_t TemporalProfTracesSectionStart = 0;609 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {610 TemporalProfTracesSectionStart = OS.tell();611 OS.write(TemporalProfTraces.size());612 OS.write(TemporalProfTraceStreamSize);613 for (auto &Trace : TemporalProfTraces) {614 OS.write(Trace.Weight);615 OS.write(Trace.FunctionNameRefs.size());616 for (auto &NameRef : Trace.FunctionNameRefs)617 OS.write(NameRef);618 }619 }620 621 // Allocate space for data to be serialized out.622 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =623 IndexedInstrProf::allocSummary(SummarySize);624 // Compute the Summary and copy the data to the data625 // structure to be serialized out (to disk or buffer).626 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();627 setSummary(TheSummary.get(), *PS);628 InfoObj->SummaryBuilder = nullptr;629 630 // For Context Sensitive summary.631 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;632 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {633 TheCSSummary = IndexedInstrProf::allocSummary(SummarySize);634 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();635 setSummary(TheCSSummary.get(), *CSPS);636 }637 InfoObj->CSSummaryBuilder = nullptr;638 639 SmallVector<uint64_t, 8> HeaderOffsets = {HashTableStart, MemProfSectionStart,640 BinaryIdSectionStart,641 TemporalProfTracesSectionStart};642 if (!WritePrevVersion)643 HeaderOffsets.push_back(VTableNamesSectionStart);644 645 PatchItem PatchItems[] = {646 // Patch the Header fields647 {BackPatchStartOffset, HeaderOffsets},648 // Patch the summary data.649 {SummaryOffset,650 ArrayRef<uint64_t>(reinterpret_cast<uint64_t *>(TheSummary.get()),651 SummarySize / sizeof(uint64_t))},652 {CSSummaryOffset,653 ArrayRef<uint64_t>(reinterpret_cast<uint64_t *>(TheCSSummary.get()),654 CSSummarySize)}};655 656 OS.patch(PatchItems);657 658 for (const auto &I : FunctionData)659 for (const auto &F : I.getValue())660 if (Error E = validateRecord(F.second))661 return E;662 663 return Error::success();664}665 666Error InstrProfWriter::write(raw_fd_ostream &OS) {667 // Write the hash table.668 ProfOStream POS(OS);669 return writeImpl(POS);670}671 672Error InstrProfWriter::write(raw_string_ostream &OS) {673 ProfOStream POS(OS);674 return writeImpl(POS);675}676 677std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {678 std::string Data;679 raw_string_ostream OS(Data);680 // Write the hash table.681 if (Error E = write(OS))682 return nullptr;683 // Return this in an aligned memory buffer.684 return MemoryBuffer::getMemBufferCopy(Data);685}686 687static const char *ValueProfKindStr[] = {688#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,689#include "llvm/ProfileData/InstrProfData.inc"690};691 692Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) {693 for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {694 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)695 continue;696 uint32_t NS = Func.getNumValueSites(VK);697 for (uint32_t S = 0; S < NS; S++) {698 DenseSet<uint64_t> SeenValues;699 for (const auto &V : Func.getValueArrayForSite(VK, S))700 if (!SeenValues.insert(V.Value).second)701 return make_error<InstrProfError>(instrprof_error::invalid_prof);702 }703 }704 705 return Error::success();706}707 708void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,709 const InstrProfRecord &Func,710 InstrProfSymtab &Symtab,711 raw_fd_ostream &OS) {712 OS << Name << "\n";713 OS << "# Func Hash:\n" << Hash << "\n";714 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";715 OS << "# Counter Values:\n";716 for (uint64_t Count : Func.Counts)717 OS << Count << "\n";718 719 if (Func.BitmapBytes.size() > 0) {720 OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n";721 OS << "# Bitmap Byte Values:\n";722 for (uint8_t Byte : Func.BitmapBytes) {723 OS << "0x";724 OS.write_hex(Byte);725 OS << "\n";726 }727 OS << "\n";728 }729 730 uint32_t NumValueKinds = Func.getNumValueKinds();731 if (!NumValueKinds) {732 OS << "\n";733 return;734 }735 736 OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";737 for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {738 uint32_t NS = Func.getNumValueSites(VK);739 if (!NS)740 continue;741 OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";742 OS << "# NumValueSites:\n" << NS << "\n";743 for (uint32_t S = 0; S < NS; S++) {744 auto VD = Func.getValueArrayForSite(VK, S);745 OS << VD.size() << "\n";746 for (const auto &V : VD) {747 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)748 OS << Symtab.getFuncOrVarNameIfDefined(V.Value) << ":" << V.Count749 << "\n";750 else751 OS << V.Value << ":" << V.Count << "\n";752 }753 }754 }755 756 OS << "\n";757}758 759Error InstrProfWriter::writeText(raw_fd_ostream &OS) {760 // Check CS first since it implies an IR level profile.761 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))762 OS << "# CSIR level Instrumentation Flag\n:csir\n";763 else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))764 OS << "# IR level Instrumentation Flag\n:ir\n";765 766 if (static_cast<bool>(ProfileKind &767 InstrProfKind::FunctionEntryInstrumentation))768 OS << "# Always instrument the function entry block\n:entry_first\n";769 if (static_cast<bool>(ProfileKind &770 InstrProfKind::LoopEntriesInstrumentation))771 OS << "# Always instrument the loop entry "772 "blocks\n:instrument_loop_entries\n";773 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))774 OS << "# Instrument block coverage\n:single_byte_coverage\n";775 InstrProfSymtab Symtab;776 777 using FuncPair = detail::DenseMapPair<uint64_t, InstrProfRecord>;778 using RecordType = std::pair<StringRef, FuncPair>;779 SmallVector<RecordType, 4> OrderedFuncData;780 781 for (const auto &I : FunctionData) {782 if (shouldEncodeData(I.getValue())) {783 if (Error E = Symtab.addFuncName(I.getKey()))784 return E;785 for (const auto &Func : I.getValue())786 OrderedFuncData.push_back(std::make_pair(I.getKey(), Func));787 }788 }789 790 for (const auto &VTableName : VTableNames)791 if (Error E = Symtab.addVTableName(VTableName.getKey()))792 return E;793 794 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))795 writeTextTemporalProfTraceData(OS, Symtab);796 797 llvm::sort(OrderedFuncData, [](const RecordType &A, const RecordType &B) {798 return std::tie(A.first, A.second.first) <799 std::tie(B.first, B.second.first);800 });801 802 for (const auto &record : OrderedFuncData) {803 const StringRef &Name = record.first;804 const FuncPair &Func = record.second;805 writeRecordInText(Name, Func.first, Func.second, Symtab, OS);806 }807 808 for (const auto &record : OrderedFuncData) {809 const FuncPair &Func = record.second;810 if (Error E = validateRecord(Func.second))811 return E;812 }813 814 return Error::success();815}816 817void InstrProfWriter::writeTextTemporalProfTraceData(raw_fd_ostream &OS,818 InstrProfSymtab &Symtab) {819 OS << ":temporal_prof_traces\n";820 OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";821 OS << "# Temporal Profile Trace Stream Size:\n"822 << TemporalProfTraceStreamSize << "\n";823 for (auto &Trace : TemporalProfTraces) {824 OS << "# Weight:\n" << Trace.Weight << "\n";825 for (auto &NameRef : Trace.FunctionNameRefs)826 OS << Symtab.getFuncOrVarName(NameRef) << ",";827 OS << "\n";828 }829 OS << "\n";830}831