299 lines · cpp
1//===- PGOCtxProfWriter.cpp - Contextual Instrumentation profile writer ---===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Write a contextual profile to bitstream.10//11//===----------------------------------------------------------------------===//12 13#include "llvm/ProfileData/PGOCtxProfWriter.h"14#include "llvm/Bitstream/BitCodeEnums.h"15#include "llvm/ProfileData/CtxInstrContextNode.h"16#include "llvm/Support/CommandLine.h"17#include "llvm/Support/Error.h"18#include "llvm/Support/YAMLTraits.h"19#include "llvm/Support/raw_ostream.h"20 21using namespace llvm;22using namespace llvm::ctx_profile;23 24static cl::opt<bool>25 IncludeEmptyOpt("ctx-prof-include-empty", cl::init(false),26 cl::desc("Also write profiles with all-zero counters. "27 "Intended for testing/debugging."));28 29PGOCtxProfileWriter::PGOCtxProfileWriter(30 raw_ostream &Out, std::optional<unsigned> VersionOverride,31 bool IncludeEmpty)32 : Writer(Out, 0),33 IncludeEmpty(IncludeEmptyOpt.getNumOccurrences() > 0 ? IncludeEmptyOpt34 : IncludeEmpty) {35 static_assert(ContainerMagic.size() == 4);36 Out.write(ContainerMagic.data(), ContainerMagic.size());37 Writer.EnterBlockInfoBlock();38 {39 auto DescribeBlock = [&](unsigned ID, StringRef Name) {40 Writer.EmitRecord(bitc::BLOCKINFO_CODE_SETBID,41 SmallVector<unsigned, 1>{ID});42 Writer.EmitRecord(bitc::BLOCKINFO_CODE_BLOCKNAME,43 llvm::arrayRefFromStringRef(Name));44 };45 SmallVector<uint64_t, 16> Data;46 auto DescribeRecord = [&](unsigned RecordID, StringRef Name) {47 Data.clear();48 Data.push_back(RecordID);49 llvm::append_range(Data, Name);50 Writer.EmitRecord(bitc::BLOCKINFO_CODE_SETRECORDNAME, Data);51 };52 DescribeBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID, "Metadata");53 DescribeRecord(PGOCtxProfileRecords::Version, "Version");54 DescribeBlock(PGOCtxProfileBlockIDs::ContextsSectionBlockID, "Contexts");55 DescribeBlock(PGOCtxProfileBlockIDs::ContextRootBlockID, "Root");56 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");57 DescribeRecord(PGOCtxProfileRecords::TotalRootEntryCount,58 "TotalRootEntryCount");59 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");60 DescribeBlock(PGOCtxProfileBlockIDs::UnhandledBlockID, "Unhandled");61 DescribeBlock(PGOCtxProfileBlockIDs::ContextNodeBlockID, "Context");62 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");63 DescribeRecord(PGOCtxProfileRecords::CallsiteIndex, "CalleeIndex");64 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");65 DescribeBlock(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID,66 "FlatProfiles");67 DescribeBlock(PGOCtxProfileBlockIDs::FlatProfileBlockID, "Flat");68 DescribeRecord(PGOCtxProfileRecords::Guid, "GUID");69 DescribeRecord(PGOCtxProfileRecords::Counters, "Counters");70 }71 Writer.ExitBlock();72 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID, CodeLen);73 const auto Version = VersionOverride.value_or(CurrentVersion);74 Writer.EmitRecord(PGOCtxProfileRecords::Version,75 SmallVector<unsigned, 1>({Version}));76}77 78void PGOCtxProfileWriter::writeCounters(ArrayRef<uint64_t> Counters) {79 Writer.EmitCode(bitc::UNABBREV_RECORD);80 Writer.EmitVBR(PGOCtxProfileRecords::Counters, VBREncodingBits);81 Writer.EmitVBR(Counters.size(), VBREncodingBits);82 for (uint64_t C : Counters)83 Writer.EmitVBR64(C, VBREncodingBits);84}85 86void PGOCtxProfileWriter::writeGuid(ctx_profile::GUID Guid) {87 Writer.EmitRecord(PGOCtxProfileRecords::Guid, SmallVector<uint64_t, 1>{Guid});88}89 90void PGOCtxProfileWriter::writeCallsiteIndex(uint32_t CallsiteIndex) {91 Writer.EmitRecord(PGOCtxProfileRecords::CallsiteIndex,92 SmallVector<uint64_t, 1>{CallsiteIndex});93}94 95void PGOCtxProfileWriter::writeRootEntryCount(uint64_t TotalRootEntryCount) {96 Writer.EmitRecord(PGOCtxProfileRecords::TotalRootEntryCount,97 SmallVector<uint64_t, 1>{TotalRootEntryCount});98}99 100// recursively write all the subcontexts. We do need to traverse depth first to101// model the context->subcontext implicitly, and since this captures call102// stacks, we don't really need to be worried about stack overflow and we can103// keep the implementation simple.104void PGOCtxProfileWriter::writeNode(uint32_t CallsiteIndex,105 const ContextNode &Node) {106 // A node with no counters is an error. We don't expect this to happen from107 // the runtime, rather, this is interesting for testing the reader.108 if (!IncludeEmpty && (Node.counters_size() > 0 && Node.entrycount() == 0))109 return;110 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextNodeBlockID, CodeLen);111 writeGuid(Node.guid());112 writeCallsiteIndex(CallsiteIndex);113 writeCounters({Node.counters(), Node.counters_size()});114 writeSubcontexts(Node);115 Writer.ExitBlock();116}117 118void PGOCtxProfileWriter::writeSubcontexts(const ContextNode &Node) {119 for (uint32_t I = 0U; I < Node.callsites_size(); ++I)120 for (const auto *Subcontext = Node.subContexts()[I]; Subcontext;121 Subcontext = Subcontext->next())122 writeNode(I, *Subcontext);123}124 125void PGOCtxProfileWriter::startContextSection() {126 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextsSectionBlockID, CodeLen);127}128 129void PGOCtxProfileWriter::startFlatSection() {130 Writer.EnterSubblock(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID,131 CodeLen);132}133 134void PGOCtxProfileWriter::endContextSection() { Writer.ExitBlock(); }135void PGOCtxProfileWriter::endFlatSection() { Writer.ExitBlock(); }136 137void PGOCtxProfileWriter::writeContextual(const ContextNode &RootNode,138 const ContextNode *Unhandled,139 uint64_t TotalRootEntryCount) {140 if (!IncludeEmpty && (!TotalRootEntryCount || (RootNode.counters_size() > 0 &&141 RootNode.entrycount() == 0)))142 return;143 Writer.EnterSubblock(PGOCtxProfileBlockIDs::ContextRootBlockID, CodeLen);144 writeGuid(RootNode.guid());145 writeRootEntryCount(TotalRootEntryCount);146 writeCounters({RootNode.counters(), RootNode.counters_size()});147 148 Writer.EnterSubblock(PGOCtxProfileBlockIDs::UnhandledBlockID, CodeLen);149 for (const auto *P = Unhandled; P; P = P->next())150 writeFlat(P->guid(), P->counters(), P->counters_size());151 Writer.ExitBlock();152 153 writeSubcontexts(RootNode);154 Writer.ExitBlock();155}156 157void PGOCtxProfileWriter::writeFlat(ctx_profile::GUID Guid,158 const uint64_t *Buffer, size_t Size) {159 Writer.EnterSubblock(PGOCtxProfileBlockIDs::FlatProfileBlockID, CodeLen);160 writeGuid(Guid);161 writeCounters({Buffer, Size});162 Writer.ExitBlock();163}164 165namespace {166 167/// Representation of the context node suitable for yaml serialization /168/// deserialization.169using SerializableFlatProfileRepresentation =170 std::pair<ctx_profile::GUID, std::vector<uint64_t>>;171 172struct SerializableCtxRepresentation {173 ctx_profile::GUID Guid = 0;174 std::vector<uint64_t> Counters;175 std::vector<std::vector<SerializableCtxRepresentation>> Callsites;176};177 178struct SerializableRootRepresentation : public SerializableCtxRepresentation {179 uint64_t TotalRootEntryCount = 0;180 std::vector<SerializableFlatProfileRepresentation> Unhandled;181};182 183struct SerializableProfileRepresentation {184 std::vector<SerializableRootRepresentation> Contexts;185 std::vector<SerializableFlatProfileRepresentation> FlatProfiles;186};187 188ctx_profile::ContextNode *189createNode(std::vector<std::unique_ptr<char[]>> &Nodes,190 const std::vector<SerializableCtxRepresentation> &DCList);191 192// Convert a DeserializableCtx into a ContextNode, potentially linking it to193// its sibling (e.g. callee at same callsite) "Next".194ctx_profile::ContextNode *195createNode(std::vector<std::unique_ptr<char[]>> &Nodes,196 const SerializableCtxRepresentation &DC,197 ctx_profile::ContextNode *Next = nullptr) {198 auto AllocSize = ctx_profile::ContextNode::getAllocSize(DC.Counters.size(),199 DC.Callsites.size());200 auto *Mem = Nodes.emplace_back(std::make_unique<char[]>(AllocSize)).get();201 std::memset(Mem, 0, AllocSize);202 auto *Ret = new (Mem) ctx_profile::ContextNode(DC.Guid, DC.Counters.size(),203 DC.Callsites.size(), Next);204 std::memcpy(Ret->counters(), DC.Counters.data(),205 sizeof(uint64_t) * DC.Counters.size());206 for (const auto &[I, DCList] : llvm::enumerate(DC.Callsites))207 Ret->subContexts()[I] = createNode(Nodes, DCList);208 return Ret;209}210 211// Convert a list of SerializableCtxRepresentation into a linked list of212// ContextNodes.213ctx_profile::ContextNode *214createNode(std::vector<std::unique_ptr<char[]>> &Nodes,215 const std::vector<SerializableCtxRepresentation> &DCList) {216 ctx_profile::ContextNode *List = nullptr;217 for (const auto &DC : DCList)218 List = createNode(Nodes, DC, List);219 return List;220}221} // namespace222 223LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableCtxRepresentation)224LLVM_YAML_IS_SEQUENCE_VECTOR(std::vector<SerializableCtxRepresentation>)225LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableRootRepresentation)226LLVM_YAML_IS_SEQUENCE_VECTOR(SerializableFlatProfileRepresentation)227template <> struct yaml::MappingTraits<SerializableCtxRepresentation> {228 static void mapping(yaml::IO &IO, SerializableCtxRepresentation &SCR) {229 IO.mapRequired("Guid", SCR.Guid);230 IO.mapRequired("Counters", SCR.Counters);231 IO.mapOptional("Callsites", SCR.Callsites);232 }233};234 235template <> struct yaml::MappingTraits<SerializableRootRepresentation> {236 static void mapping(yaml::IO &IO, SerializableRootRepresentation &R) {237 yaml::MappingTraits<SerializableCtxRepresentation>::mapping(IO, R);238 IO.mapRequired("TotalRootEntryCount", R.TotalRootEntryCount);239 IO.mapOptional("Unhandled", R.Unhandled);240 }241};242 243template <> struct yaml::MappingTraits<SerializableProfileRepresentation> {244 static void mapping(yaml::IO &IO, SerializableProfileRepresentation &SPR) {245 IO.mapOptional("Contexts", SPR.Contexts);246 IO.mapOptional("FlatProfiles", SPR.FlatProfiles);247 }248};249 250template <> struct yaml::MappingTraits<SerializableFlatProfileRepresentation> {251 static void mapping(yaml::IO &IO,252 SerializableFlatProfileRepresentation &SFPR) {253 IO.mapRequired("Guid", SFPR.first);254 IO.mapRequired("Counters", SFPR.second);255 }256};257 258Error llvm::createCtxProfFromYAML(StringRef Profile, raw_ostream &Out) {259 yaml::Input In(Profile);260 SerializableProfileRepresentation SPR;261 In >> SPR;262 if (In.error())263 return createStringError(In.error(), "incorrect yaml content");264 std::vector<std::unique_ptr<char[]>> Nodes;265 std::error_code EC;266 if (EC)267 return createStringError(EC, "failed to open output");268 PGOCtxProfileWriter Writer(Out);269 270 if (!SPR.Contexts.empty()) {271 Writer.startContextSection();272 for (const auto &DC : SPR.Contexts) {273 auto *TopList = createNode(Nodes, DC);274 if (!TopList)275 return createStringError(276 "Unexpected error converting internal structure to ctx profile");277 278 ctx_profile::ContextNode *FirstUnhandled = nullptr;279 for (const auto &U : DC.Unhandled) {280 SerializableCtxRepresentation Unhandled;281 Unhandled.Guid = U.first;282 Unhandled.Counters = U.second;283 FirstUnhandled = createNode(Nodes, Unhandled, FirstUnhandled);284 }285 Writer.writeContextual(*TopList, FirstUnhandled, DC.TotalRootEntryCount);286 }287 Writer.endContextSection();288 }289 if (!SPR.FlatProfiles.empty()) {290 Writer.startFlatSection();291 for (const auto &[Guid, Counters] : SPR.FlatProfiles)292 Writer.writeFlat(Guid, Counters.data(), Counters.size());293 Writer.endFlatSection();294 }295 if (EC)296 return createStringError(EC, "failed to write output");297 return Error::success();298}299