394 lines · cpp
1//===- PGOCtxProfReader.cpp - Contextual Instrumentation profile reader ---===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Read a contextual profile into a datastructure suitable for maintenance10// throughout IPO11//12//===----------------------------------------------------------------------===//13 14#include "llvm/ProfileData/PGOCtxProfReader.h"15#include "llvm/Bitstream/BitCodeEnums.h"16#include "llvm/Bitstream/BitstreamReader.h"17#include "llvm/ProfileData/InstrProf.h"18#include "llvm/ProfileData/PGOCtxProfWriter.h"19#include "llvm/Support/Error.h"20#include "llvm/Support/ErrorHandling.h"21#include "llvm/Support/YAMLTraits.h"22#include <utility>23 24using namespace llvm;25 26// FIXME(#92054) - these Error handling macros are (re-)invented in a few27// places.28#define EXPECT_OR_RET(LHS, RHS) \29 auto LHS = RHS; \30 if (!LHS) \31 return LHS.takeError();32 33#define RET_ON_ERR(EXPR) \34 if (auto Err = (EXPR)) \35 return Err;36 37Expected<PGOCtxProfContext &>38PGOCtxProfContext::getOrEmplace(uint32_t Index, GlobalValue::GUID G,39 SmallVectorImpl<uint64_t> &&Counters) {40 auto [Iter, Inserted] =41 Callsites[Index].insert({G, PGOCtxProfContext(G, std::move(Counters))});42 if (!Inserted)43 return make_error<InstrProfError>(instrprof_error::invalid_prof,44 "Duplicate GUID for same callsite.");45 return Iter->second;46}47 48Expected<BitstreamEntry> PGOCtxProfileReader::advance() {49 return Cursor.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs);50}51 52Error PGOCtxProfileReader::wrongValue(const Twine &Msg) {53 return make_error<InstrProfError>(instrprof_error::invalid_prof, Msg);54}55 56Error PGOCtxProfileReader::unsupported(const Twine &Msg) {57 return make_error<InstrProfError>(instrprof_error::unsupported_version, Msg);58}59 60bool PGOCtxProfileReader::tryGetNextKnownBlockID(PGOCtxProfileBlockIDs &ID) {61 auto Blk = advance();62 if (!Blk) {63 consumeError(Blk.takeError());64 return false;65 }66 if (Blk->Kind != BitstreamEntry::SubBlock)67 return false;68 if (PGOCtxProfileBlockIDs::FIRST_VALID > Blk->ID ||69 PGOCtxProfileBlockIDs::LAST_VALID < Blk->ID)70 return false;71 ID = static_cast<PGOCtxProfileBlockIDs>(Blk->ID);72 return true;73}74 75bool PGOCtxProfileReader::canEnterBlockWithID(PGOCtxProfileBlockIDs ID) {76 PGOCtxProfileBlockIDs Test = {};77 return tryGetNextKnownBlockID(Test) && Test == ID;78}79 80Error PGOCtxProfileReader::enterBlockWithID(PGOCtxProfileBlockIDs ID) {81 RET_ON_ERR(Cursor.EnterSubBlock(ID));82 return Error::success();83}84 85// Note: we use PGOCtxProfContext for flat profiles also, as the latter are86// structurally similar. Alternative modeling here seems a bit overkill at the87// moment.88Expected<std::pair<std::optional<uint32_t>, PGOCtxProfContext>>89PGOCtxProfileReader::readProfile(PGOCtxProfileBlockIDs Kind) {90 assert((Kind == PGOCtxProfileBlockIDs::ContextRootBlockID ||91 Kind == PGOCtxProfileBlockIDs::ContextNodeBlockID ||92 Kind == PGOCtxProfileBlockIDs::FlatProfileBlockID) &&93 "Unexpected profile kind");94 RET_ON_ERR(enterBlockWithID(Kind));95 96 std::optional<ctx_profile::GUID> Guid;97 std::optional<SmallVector<uint64_t, 16>> Counters;98 std::optional<uint32_t> CallsiteIndex;99 std::optional<uint64_t> TotalEntryCount;100 std::optional<CtxProfFlatProfile> Unhandled;101 SmallVector<uint64_t, 1> RecordValues;102 103 const bool ExpectIndex = Kind == PGOCtxProfileBlockIDs::ContextNodeBlockID;104 const bool IsRoot = Kind == PGOCtxProfileBlockIDs::ContextRootBlockID;105 // We don't prescribe the order in which the records come in, and we are ok106 // if other unsupported records appear. We seek in the current subblock until107 // we get all we know.108 auto GotAllWeNeed = [&]() {109 return Guid.has_value() && Counters.has_value() &&110 (!ExpectIndex || CallsiteIndex.has_value()) &&111 (!IsRoot || TotalEntryCount.has_value()) &&112 (!IsRoot || Unhandled.has_value());113 };114 115 while (!GotAllWeNeed()) {116 RecordValues.clear();117 EXPECT_OR_RET(Entry, advance());118 if (Entry->Kind != BitstreamEntry::Record) {119 if (IsRoot && Entry->Kind == BitstreamEntry::SubBlock &&120 Entry->ID == PGOCtxProfileBlockIDs::UnhandledBlockID) {121 RET_ON_ERR(enterBlockWithID(PGOCtxProfileBlockIDs::UnhandledBlockID));122 Unhandled = CtxProfFlatProfile();123 RET_ON_ERR(loadFlatProfileList(*Unhandled));124 continue;125 }126 return wrongValue(127 "Expected records before encountering more subcontexts");128 }129 EXPECT_OR_RET(ReadRecord,130 Cursor.readRecord(bitc::UNABBREV_RECORD, RecordValues));131 switch (*ReadRecord) {132 case PGOCtxProfileRecords::Guid:133 if (RecordValues.size() != 1)134 return wrongValue("The GUID record should have exactly one value");135 Guid = RecordValues[0];136 break;137 case PGOCtxProfileRecords::Counters:138 Counters = std::move(RecordValues);139 if (Counters->empty())140 return wrongValue("Empty counters. At least the entry counter (one "141 "value) was expected");142 break;143 case PGOCtxProfileRecords::CallsiteIndex:144 if (!ExpectIndex)145 return wrongValue("The root context should not have a callee index");146 if (RecordValues.size() != 1)147 return wrongValue("The callee index should have exactly one value");148 CallsiteIndex = RecordValues[0];149 break;150 case PGOCtxProfileRecords::TotalRootEntryCount:151 if (!IsRoot)152 return wrongValue("Non-root has a total entry count record");153 if (RecordValues.size() != 1)154 return wrongValue(155 "The root total entry count record should have exactly one value");156 TotalEntryCount = RecordValues[0];157 break;158 default:159 // OK if we see records we do not understand, like records (profile160 // components) introduced later.161 break;162 }163 }164 165 PGOCtxProfContext Ret(*Guid, std::move(*Counters), TotalEntryCount,166 std::move(Unhandled));167 168 while (canEnterBlockWithID(PGOCtxProfileBlockIDs::ContextNodeBlockID)) {169 EXPECT_OR_RET(SC, readProfile(PGOCtxProfileBlockIDs::ContextNodeBlockID));170 auto &Targets = Ret.callsites()[*SC->first];171 auto [_, Inserted] =172 Targets.insert({SC->second.guid(), std::move(SC->second)});173 if (!Inserted)174 return wrongValue(175 "Unexpected duplicate target (callee) at the same callsite.");176 }177 return std::make_pair(CallsiteIndex, std::move(Ret));178}179 180Error PGOCtxProfileReader::readMetadata() {181 if (Magic.size() < PGOCtxProfileWriter::ContainerMagic.size() ||182 Magic != PGOCtxProfileWriter::ContainerMagic)183 return make_error<InstrProfError>(instrprof_error::invalid_prof,184 "Invalid magic");185 186 BitstreamEntry Entry;187 RET_ON_ERR(Cursor.advance().moveInto(Entry));188 if (Entry.Kind != BitstreamEntry::SubBlock ||189 Entry.ID != bitc::BLOCKINFO_BLOCK_ID)190 return unsupported("Expected Block ID");191 // We don't need the blockinfo to read the rest, it's metadata usable for e.g.192 // llvm-bcanalyzer.193 RET_ON_ERR(Cursor.SkipBlock());194 195 EXPECT_OR_RET(Blk, advance());196 if (Blk->Kind != BitstreamEntry::SubBlock)197 return unsupported("Expected Version record");198 RET_ON_ERR(199 Cursor.EnterSubBlock(PGOCtxProfileBlockIDs::ProfileMetadataBlockID));200 EXPECT_OR_RET(MData, advance());201 if (MData->Kind != BitstreamEntry::Record)202 return unsupported("Expected Version record");203 204 SmallVector<uint64_t, 1> Ver;205 EXPECT_OR_RET(Code, Cursor.readRecord(bitc::UNABBREV_RECORD, Ver));206 if (*Code != PGOCtxProfileRecords::Version)207 return unsupported("Expected Version record");208 if (Ver.size() != 1 || Ver[0] > PGOCtxProfileWriter::CurrentVersion)209 return unsupported("Version " + Twine(*Code) +210 " is higher than supported version " +211 Twine(PGOCtxProfileWriter::CurrentVersion));212 return Error::success();213}214 215Error PGOCtxProfileReader::loadContexts(CtxProfContextualProfiles &P) {216 RET_ON_ERR(enterBlockWithID(PGOCtxProfileBlockIDs::ContextsSectionBlockID));217 while (canEnterBlockWithID(PGOCtxProfileBlockIDs::ContextRootBlockID)) {218 EXPECT_OR_RET(E, readProfile(PGOCtxProfileBlockIDs::ContextRootBlockID));219 auto Key = E->second.guid();220 if (!P.insert({Key, std::move(E->second)}).second)221 return wrongValue("Duplicate roots");222 }223 return Error::success();224}225 226Error PGOCtxProfileReader::loadFlatProfileList(CtxProfFlatProfile &P) {227 while (canEnterBlockWithID(PGOCtxProfileBlockIDs::FlatProfileBlockID)) {228 EXPECT_OR_RET(E, readProfile(PGOCtxProfileBlockIDs::FlatProfileBlockID));229 auto Guid = E->second.guid();230 if (!P.insert({Guid, std::move(E->second.counters())}).second)231 return wrongValue("Duplicate flat profile entries");232 }233 return Error::success();234}235 236Error PGOCtxProfileReader::loadFlatProfiles(CtxProfFlatProfile &P) {237 RET_ON_ERR(238 enterBlockWithID(PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID));239 return loadFlatProfileList(P);240}241 242Expected<PGOCtxProfile> PGOCtxProfileReader::loadProfiles() {243 RET_ON_ERR(readMetadata());244 PGOCtxProfile Ret;245 PGOCtxProfileBlockIDs Test = {};246 for (auto I = 0; I < 2; ++I) {247 if (!tryGetNextKnownBlockID(Test))248 break;249 if (Test == PGOCtxProfileBlockIDs::ContextsSectionBlockID) {250 RET_ON_ERR(loadContexts(Ret.Contexts));251 } else if (Test == PGOCtxProfileBlockIDs::FlatProfilesSectionBlockID) {252 RET_ON_ERR(loadFlatProfiles(Ret.FlatProfiles));253 } else {254 return wrongValue("Unexpected section");255 }256 }257 258 return std::move(Ret);259}260 261namespace {262// We want to pass `const` values PGOCtxProfContext references to the yaml263// converter, and the regular yaml mapping APIs are designed to handle both264// serialization and deserialization, which prevents using const for265// serialization. Using an intermediate datastructure is overkill, both266// space-wise and design complexity-wise. Instead, we use the lower-level APIs.267void toYaml(yaml::Output &Out, const PGOCtxProfContext &Ctx);268 269void toYaml(yaml::Output &Out,270 const PGOCtxProfContext::CallTargetMapTy &CallTargets) {271 Out.beginSequence();272 size_t Index = 0;273 void *SaveData = nullptr;274 for (const auto &[_, Ctx] : CallTargets) {275 Out.preflightElement(Index++, SaveData);276 toYaml(Out, Ctx);277 Out.postflightElement(nullptr);278 }279 Out.endSequence();280}281 282void toYaml(yaml::Output &Out,283 const PGOCtxProfContext::CallsiteMapTy &Callsites) {284 auto AllCS = ::llvm::make_first_range(Callsites);285 auto MaxIt = ::llvm::max_element(AllCS);286 assert(MaxIt != AllCS.end() && "We should have a max value because the "287 "callsites collection is not empty.");288 void *SaveData = nullptr;289 Out.beginSequence();290 for (auto I = 0U; I <= *MaxIt; ++I) {291 Out.preflightElement(I, SaveData);292 auto It = Callsites.find(I);293 if (It == Callsites.end()) {294 // This will produce a `[ ]` sequence, which is what we want here.295 Out.beginFlowSequence();296 Out.endFlowSequence();297 } else {298 toYaml(Out, It->second);299 }300 Out.postflightElement(nullptr);301 }302 Out.endSequence();303}304 305void toYaml(yaml::Output &Out, const CtxProfFlatProfile &Flat);306 307void toYaml(yaml::Output &Out, GlobalValue::GUID Guid,308 const SmallVectorImpl<uint64_t> &Counters,309 const PGOCtxProfContext::CallsiteMapTy &Callsites,310 std::optional<uint64_t> TotalRootEntryCount = std::nullopt,311 CtxProfFlatProfile Unhandled = {}) {312 yaml::EmptyContext Empty;313 Out.beginMapping();314 void *SaveInfo = nullptr;315 bool UseDefault = false;316 {317 Out.preflightKey("Guid", /*Required=*/true, /*SameAsDefault=*/false,318 UseDefault, SaveInfo);319 yaml::yamlize(Out, Guid, true, Empty);320 Out.postflightKey(nullptr);321 }322 if (TotalRootEntryCount) {323 Out.preflightKey("TotalRootEntryCount", true, false, UseDefault, SaveInfo);324 yaml::yamlize(Out, *TotalRootEntryCount, true, Empty);325 Out.postflightKey(nullptr);326 }327 {328 Out.preflightKey("Counters", true, false, UseDefault, SaveInfo);329 Out.beginFlowSequence();330 for (size_t I = 0U, E = Counters.size(); I < E; ++I) {331 Out.preflightFlowElement(I, SaveInfo);332 uint64_t V = Counters[I];333 yaml::yamlize(Out, V, true, Empty);334 Out.postflightFlowElement(SaveInfo);335 }336 Out.endFlowSequence();337 Out.postflightKey(nullptr);338 }339 340 if (!Unhandled.empty()) {341 assert(TotalRootEntryCount.has_value());342 Out.preflightKey("Unhandled", false, false, UseDefault, SaveInfo);343 toYaml(Out, Unhandled);344 Out.postflightKey(nullptr);345 }346 347 if (!Callsites.empty()) {348 Out.preflightKey("Callsites", true, false, UseDefault, SaveInfo);349 toYaml(Out, Callsites);350 Out.postflightKey(nullptr);351 }352 Out.endMapping();353}354 355void toYaml(yaml::Output &Out, const CtxProfFlatProfile &Flat) {356 void *SaveInfo = nullptr;357 Out.beginSequence();358 size_t ElemID = 0;359 for (const auto &[Guid, Counters] : Flat) {360 Out.preflightElement(ElemID++, SaveInfo);361 toYaml(Out, Guid, Counters, {});362 Out.postflightElement(nullptr);363 }364 Out.endSequence();365}366 367void toYaml(yaml::Output &Out, const PGOCtxProfContext &Ctx) {368 if (Ctx.isRoot())369 toYaml(Out, Ctx.guid(), Ctx.counters(), Ctx.callsites(),370 Ctx.getTotalRootEntryCount(), Ctx.getUnhandled());371 else372 toYaml(Out, Ctx.guid(), Ctx.counters(), Ctx.callsites());373}374 375} // namespace376 377void llvm::convertCtxProfToYaml(raw_ostream &OS, const PGOCtxProfile &Profile) {378 yaml::Output Out(OS);379 void *SaveInfo = nullptr;380 bool UseDefault = false;381 Out.beginMapping();382 if (!Profile.Contexts.empty()) {383 Out.preflightKey("Contexts", false, false, UseDefault, SaveInfo);384 toYaml(Out, Profile.Contexts);385 Out.postflightKey(nullptr);386 }387 if (!Profile.FlatProfiles.empty()) {388 Out.preflightKey("FlatProfiles", false, false, UseDefault, SaveInfo);389 toYaml(Out, Profile.FlatProfiles);390 Out.postflightKey(nullptr);391 }392 Out.endMapping();393}394