1899 lines · cpp
1//===- unittest/ProfileData/InstrProfTest.cpp -------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/ADT/STLExtras.h"10#include "llvm/IR/DerivedTypes.h"11#include "llvm/IR/Function.h"12#include "llvm/IR/IRBuilder.h"13#include "llvm/IR/LLVMContext.h"14#include "llvm/IR/Module.h"15#include "llvm/ProfileData/IndexedMemProfData.h"16#include "llvm/ProfileData/InstrProfReader.h"17#include "llvm/ProfileData/InstrProfWriter.h"18#include "llvm/ProfileData/MemProf.h"19#include "llvm/ProfileData/MemProfData.inc"20#include "llvm/ProfileData/MemProfRadixTree.h"21#include "llvm/Support/Compression.h"22#include "llvm/Support/raw_ostream.h"23#include "llvm/Testing/Support/Error.h"24#include "gtest/gtest.h"25#include <cstdarg>26#include <initializer_list>27#include <optional>28 29using namespace llvm;30using ::llvm::memprof::LineLocation;31using ::testing::ElementsAre;32using ::testing::EndsWith;33using ::testing::IsSubsetOf;34using ::testing::Pair;35using ::testing::SizeIs;36using ::testing::UnorderedElementsAre;37 38[[nodiscard]] static ::testing::AssertionResult39ErrorEquals(instrprof_error Expected, Error E) {40 instrprof_error Found;41 std::string FoundMsg;42 handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {43 Found = IPE.get();44 FoundMsg = IPE.message();45 });46 if (Expected == Found)47 return ::testing::AssertionSuccess();48 return ::testing::AssertionFailure() << "error: " << FoundMsg << "\n";49}50 51namespace llvm {52bool operator==(const TemporalProfTraceTy &lhs,53 const TemporalProfTraceTy &rhs) {54 return lhs.Weight == rhs.Weight &&55 lhs.FunctionNameRefs == rhs.FunctionNameRefs;56}57} // end namespace llvm58 59namespace {60 61struct InstrProfTest : ::testing::Test {62 InstrProfWriter Writer;63 std::unique_ptr<IndexedInstrProfReader> Reader;64 65 void SetUp() override { Writer.setOutputSparse(false); }66 67 void readProfile(std::unique_ptr<MemoryBuffer> Profile,68 std::unique_ptr<MemoryBuffer> Remapping = nullptr) {69 auto ReaderOrErr = IndexedInstrProfReader::create(std::move(Profile),70 std::move(Remapping));71 EXPECT_THAT_ERROR(ReaderOrErr.takeError(), Succeeded());72 Reader = std::move(ReaderOrErr.get());73 }74};75 76struct SparseInstrProfTest : public InstrProfTest {77 void SetUp() override { Writer.setOutputSparse(true); }78};79 80struct InstrProfReaderWriterTest81 : public InstrProfTest,82 public ::testing::WithParamInterface<83 std::tuple<bool, uint64_t, llvm::endianness>> {84 void SetUp() override { Writer.setOutputSparse(std::get<0>(GetParam())); }85 void TearDown() override {86 // Reset writer value profile data endianness after each test case. Note87 // it's not necessary to reset reader value profile endianness for each test88 // case. Each test case creates a new reader; at reader initialization time,89 // it uses the endianness from hash table object (which is little by90 // default).91 Writer.setValueProfDataEndianness(llvm::endianness::little);92 }93 94 uint64_t getProfWeight() const { return std::get<1>(GetParam()); }95 96 llvm::endianness getEndianness() const { return std::get<2>(GetParam()); }97};98 99struct MaybeSparseInstrProfTest : public InstrProfTest,100 public ::testing::WithParamInterface<bool> {101 void SetUp() override { Writer.setOutputSparse(GetParam()); }102};103 104TEST_P(MaybeSparseInstrProfTest, write_and_read_empty_profile) {105 auto Profile = Writer.writeBuffer();106 readProfile(std::move(Profile));107 ASSERT_TRUE(Reader->begin() == Reader->end());108}109 110static const auto Err = [](Error E) {111 consumeError(std::move(E));112 FAIL();113};114 115TEST_P(MaybeSparseInstrProfTest, write_and_read_one_function) {116 Writer.addRecord({"foo", 0x1234, {1, 2, 3, 4}}, Err);117 auto Profile = Writer.writeBuffer();118 readProfile(std::move(Profile));119 120 auto I = Reader->begin(), E = Reader->end();121 ASSERT_TRUE(I != E);122 ASSERT_EQ(StringRef("foo"), I->Name);123 ASSERT_EQ(0x1234U, I->Hash);124 ASSERT_EQ(4U, I->Counts.size());125 ASSERT_EQ(1U, I->Counts[0]);126 ASSERT_EQ(2U, I->Counts[1]);127 ASSERT_EQ(3U, I->Counts[2]);128 ASSERT_EQ(4U, I->Counts[3]);129 ASSERT_TRUE(++I == E);130}131 132TEST_P(MaybeSparseInstrProfTest, get_instr_prof_record) {133 Writer.addRecord({"foo", 0x1234, {1, 2}}, Err);134 Writer.addRecord({"foo", 0x1235, {3, 4}}, Err);135 auto Profile = Writer.writeBuffer();136 readProfile(std::move(Profile));137 138 auto R = Reader->getInstrProfRecord("foo", 0x1234);139 EXPECT_THAT_ERROR(R.takeError(), Succeeded());140 ASSERT_EQ(2U, R->Counts.size());141 ASSERT_EQ(1U, R->Counts[0]);142 ASSERT_EQ(2U, R->Counts[1]);143 144 R = Reader->getInstrProfRecord("foo", 0x1235);145 EXPECT_THAT_ERROR(R.takeError(), Succeeded());146 ASSERT_EQ(2U, R->Counts.size());147 ASSERT_EQ(3U, R->Counts[0]);148 ASSERT_EQ(4U, R->Counts[1]);149 150 R = Reader->getInstrProfRecord("foo", 0x5678);151 ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, R.takeError()));152 153 R = Reader->getInstrProfRecord("bar", 0x1234);154 ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, R.takeError()));155}156 157TEST_P(MaybeSparseInstrProfTest, get_function_counts) {158 Writer.addRecord({"foo", 0x1234, {1, 2}}, Err);159 Writer.addRecord({"foo", 0x1235, {3, 4}}, Err);160 auto Profile = Writer.writeBuffer();161 readProfile(std::move(Profile));162 163 std::vector<uint64_t> Counts;164 EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),165 Succeeded());166 ASSERT_EQ(2U, Counts.size());167 ASSERT_EQ(1U, Counts[0]);168 ASSERT_EQ(2U, Counts[1]);169 170 EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),171 Succeeded());172 ASSERT_EQ(2U, Counts.size());173 ASSERT_EQ(3U, Counts[0]);174 ASSERT_EQ(4U, Counts[1]);175 176 Error E1 = Reader->getFunctionCounts("foo", 0x5678, Counts);177 ASSERT_TRUE(ErrorEquals(instrprof_error::hash_mismatch, std::move(E1)));178 179 Error E2 = Reader->getFunctionCounts("bar", 0x1234, Counts);180 ASSERT_TRUE(ErrorEquals(instrprof_error::unknown_function, std::move(E2)));181}182 183// Profile data is copied from general.proftext184TEST_F(InstrProfTest, get_profile_summary) {185 Writer.addRecord({"func1", 0x1234, {97531}}, Err);186 Writer.addRecord({"func2", 0x1234, {0, 0}}, Err);187 Writer.addRecord(188 {"func3",189 0x1234,190 {2305843009213693952, 1152921504606846976, 576460752303423488,191 288230376151711744, 144115188075855872, 72057594037927936}},192 Err);193 Writer.addRecord({"func4", 0x1234, {0}}, Err);194 auto Profile = Writer.writeBuffer();195 readProfile(std::move(Profile));196 197 auto VerifySummary = [](ProfileSummary &IPS) mutable {198 ASSERT_EQ(ProfileSummary::PSK_Instr, IPS.getKind());199 ASSERT_EQ(2305843009213693952U, IPS.getMaxFunctionCount());200 ASSERT_EQ(2305843009213693952U, IPS.getMaxCount());201 ASSERT_EQ(10U, IPS.getNumCounts());202 ASSERT_EQ(4539628424389557499U, IPS.getTotalCount());203 const std::vector<ProfileSummaryEntry> &Details = IPS.getDetailedSummary();204 uint32_t Cutoff = 800000;205 auto Predicate = [&Cutoff](const ProfileSummaryEntry &PE) {206 return PE.Cutoff == Cutoff;207 };208 auto EightyPerc = find_if(Details, Predicate);209 Cutoff = 900000;210 auto NinetyPerc = find_if(Details, Predicate);211 Cutoff = 950000;212 auto NinetyFivePerc = find_if(Details, Predicate);213 Cutoff = 990000;214 auto NinetyNinePerc = find_if(Details, Predicate);215 ASSERT_EQ(576460752303423488U, EightyPerc->MinCount);216 ASSERT_EQ(288230376151711744U, NinetyPerc->MinCount);217 ASSERT_EQ(288230376151711744U, NinetyFivePerc->MinCount);218 ASSERT_EQ(72057594037927936U, NinetyNinePerc->MinCount);219 };220 ProfileSummary &PS = Reader->getSummary(/* IsCS */ false);221 VerifySummary(PS);222 223 // Test that conversion of summary to and from Metadata works.224 LLVMContext Context;225 Metadata *MD = PS.getMD(Context);226 ASSERT_TRUE(MD);227 ProfileSummary *PSFromMD = ProfileSummary::getFromMD(MD);228 ASSERT_TRUE(PSFromMD);229 VerifySummary(*PSFromMD);230 delete PSFromMD;231 232 // Test that summary can be attached to and read back from module.233 Module M("my_module", Context);234 M.setProfileSummary(MD, ProfileSummary::PSK_Instr);235 MD = M.getProfileSummary(/* IsCS */ false);236 ASSERT_TRUE(MD);237 PSFromMD = ProfileSummary::getFromMD(MD);238 ASSERT_TRUE(PSFromMD);239 VerifySummary(*PSFromMD);240 delete PSFromMD;241}242 243TEST_F(InstrProfTest, test_writer_merge) {244 Writer.addRecord({"func1", 0x1234, {42}}, Err);245 246 InstrProfWriter Writer2;247 Writer2.addRecord({"func2", 0x1234, {0, 0}}, Err);248 249 Writer.mergeRecordsFromWriter(std::move(Writer2), Err);250 251 auto Profile = Writer.writeBuffer();252 readProfile(std::move(Profile));253 254 auto R = Reader->getInstrProfRecord("func1", 0x1234);255 EXPECT_THAT_ERROR(R.takeError(), Succeeded());256 ASSERT_EQ(1U, R->Counts.size());257 ASSERT_EQ(42U, R->Counts[0]);258 259 R = Reader->getInstrProfRecord("func2", 0x1234);260 EXPECT_THAT_ERROR(R.takeError(), Succeeded());261 ASSERT_EQ(2U, R->Counts.size());262 ASSERT_EQ(0U, R->Counts[0]);263 ASSERT_EQ(0U, R->Counts[1]);264}265 266TEST_F(InstrProfTest, test_merge_temporal_prof_traces_truncated) {267 uint64_t ReservoirSize = 10;268 uint64_t MaxTraceLength = 2;269 InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength);270 ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile),271 Succeeded());272 273 TemporalProfTraceTy LargeTrace, SmallTrace;274 LargeTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo"),275 IndexedInstrProf::ComputeHash("bar"),276 IndexedInstrProf::ComputeHash("goo")};277 SmallTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo"),278 IndexedInstrProf::ComputeHash("bar")};279 280 SmallVector<TemporalProfTraceTy, 4> Traces = {LargeTrace, SmallTrace};281 Writer.addTemporalProfileTraces(Traces, 2);282 283 auto Profile = Writer.writeBuffer();284 readProfile(std::move(Profile));285 286 ASSERT_TRUE(Reader->hasTemporalProfile());287 EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 2U);288 EXPECT_THAT(Reader->getTemporalProfTraces(),289 UnorderedElementsAre(SmallTrace, SmallTrace));290}291 292TEST_F(InstrProfTest, test_merge_traces_from_writer) {293 uint64_t ReservoirSize = 10;294 uint64_t MaxTraceLength = 10;295 InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength);296 InstrProfWriter Writer2(/*Sparse=*/false, ReservoirSize, MaxTraceLength);297 ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile),298 Succeeded());299 ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::TemporalProfile),300 Succeeded());301 302 TemporalProfTraceTy FooTrace, BarTrace;303 FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo")};304 BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("bar")};305 306 SmallVector<TemporalProfTraceTy, 4> Traces1({FooTrace}), Traces2({BarTrace});307 Writer.addTemporalProfileTraces(Traces1, 1);308 Writer2.addTemporalProfileTraces(Traces2, 1);309 Writer.mergeRecordsFromWriter(std::move(Writer2), Err);310 311 auto Profile = Writer.writeBuffer();312 readProfile(std::move(Profile));313 314 ASSERT_TRUE(Reader->hasTemporalProfile());315 EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 2U);316 EXPECT_THAT(Reader->getTemporalProfTraces(),317 UnorderedElementsAre(FooTrace, BarTrace));318}319 320TEST_F(InstrProfTest, test_merge_traces_sampled) {321 uint64_t ReservoirSize = 3;322 uint64_t MaxTraceLength = 10;323 InstrProfWriter Writer(/*Sparse=*/false, ReservoirSize, MaxTraceLength);324 ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::TemporalProfile),325 Succeeded());326 327 TemporalProfTraceTy FooTrace, BarTrace, GooTrace;328 FooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("foo")};329 BarTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("bar")};330 GooTrace.FunctionNameRefs = {IndexedInstrProf::ComputeHash("Goo")};331 332 // Add some sampled traces333 SmallVector<TemporalProfTraceTy, 4> SampledTraces = {FooTrace, BarTrace,334 GooTrace};335 Writer.addTemporalProfileTraces(SampledTraces, 5);336 // Add some unsampled traces337 SmallVector<TemporalProfTraceTy, 4> UnsampledTraces = {BarTrace, GooTrace};338 Writer.addTemporalProfileTraces(UnsampledTraces, 2);339 UnsampledTraces = {FooTrace};340 Writer.addTemporalProfileTraces(UnsampledTraces, 1);341 342 auto Profile = Writer.writeBuffer();343 readProfile(std::move(Profile));344 345 ASSERT_TRUE(Reader->hasTemporalProfile());346 EXPECT_EQ(Reader->getTemporalProfTraceStreamSize(), 8U);347 // Check that we have a subset of all the traces we added348 EXPECT_THAT(Reader->getTemporalProfTraces(), SizeIs(ReservoirSize));349 EXPECT_THAT(350 Reader->getTemporalProfTraces(),351 IsSubsetOf({FooTrace, BarTrace, GooTrace, BarTrace, GooTrace, FooTrace}));352}353 354using ::llvm::memprof::IndexedMemProfData;355using ::llvm::memprof::IndexedMemProfRecord;356using ::llvm::memprof::MemInfoBlock;357 358IndexedMemProfData getMemProfDataForTest() {359 IndexedMemProfData MemProfData;360 361 MemProfData.Frames.insert({0, {0x123, 1, 2, false}});362 MemProfData.Frames.insert({1, {0x345, 3, 4, true}});363 MemProfData.Frames.insert({2, {0x125, 5, 6, false}});364 MemProfData.Frames.insert({3, {0x567, 7, 8, true}});365 MemProfData.Frames.insert({4, {0x124, 5, 6, false}});366 MemProfData.Frames.insert({5, {0x789, 8, 9, true}});367 368 MemProfData.CallStacks.insert({0x111, {0, 1}});369 MemProfData.CallStacks.insert({0x222, {2, 3}});370 MemProfData.CallStacks.insert({0x333, {4, 5}});371 372 return MemProfData;373}374 375// Populate all of the fields of MIB.376MemInfoBlock makeFullMIB() {377 MemInfoBlock MIB;378#define MIBEntryDef(NameTag, Name, Type) MIB.NameTag;379#include "llvm/ProfileData/MIBEntryDef.inc"380#undef MIBEntryDef381 return MIB;382}383 384// Populate those fields returned by getHotColdSchema.385MemInfoBlock makePartialMIB() {386 MemInfoBlock MIB;387 MIB.AllocCount = 1;388 MIB.TotalSize = 5;389 MIB.TotalLifetime = 10;390 MIB.TotalLifetimeAccessDensity = 23;391 return MIB;392}393 394IndexedMemProfRecord395makeRecord(std::initializer_list<::llvm::memprof::CallStackId> AllocFrames,396 std::initializer_list<::llvm::memprof::CallStackId> CallSiteFrames,397 const MemInfoBlock &Block, const memprof::MemProfSchema &Schema) {398 IndexedMemProfRecord MR;399 for (const auto &CSId : AllocFrames)400 MR.AllocSites.emplace_back(CSId, Block, Schema);401 for (const auto &CSId : CallSiteFrames)402 MR.CallSites.push_back(llvm::memprof::IndexedCallSiteInfo(CSId));403 return MR;404}405 406MATCHER_P(EqualsRecord, Want, "") {407 const memprof::MemProfRecord &Got = arg;408 409 auto PrintAndFail = [&]() {410 std::string Buffer;411 llvm::raw_string_ostream OS(Buffer);412 OS << "Want:\n";413 Want.print(OS);414 OS << "Got:\n";415 Got.print(OS);416 *result_listener << "MemProf Record differs!\n" << Buffer;417 return false;418 };419 420 if (Want.AllocSites.size() != Got.AllocSites.size())421 return PrintAndFail();422 if (Want.CallSites.size() != Got.CallSites.size())423 return PrintAndFail();424 425 for (size_t I = 0; I < Got.AllocSites.size(); I++) {426 if (Want.AllocSites[I].Info != Got.AllocSites[I].Info)427 return PrintAndFail();428 if (Want.AllocSites[I].CallStack != Got.AllocSites[I].CallStack)429 return PrintAndFail();430 }431 432 for (size_t I = 0; I < Got.CallSites.size(); I++) {433 if (Want.CallSites[I] != Got.CallSites[I])434 return PrintAndFail();435 }436 return true;437}438 439TEST_F(InstrProfTest, test_memprof_v4_full_schema) {440 const MemInfoBlock MIB = makeFullMIB();441 442 Writer.setMemProfVersionRequested(memprof::Version4);443 Writer.setMemProfFullSchema(true);444 445 ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),446 Succeeded());447 448 const IndexedMemProfRecord IndexedMR = makeRecord(449 /*AllocFrames=*/{0x111, 0x222},450 /*CallSiteFrames=*/{0x333}, MIB, memprof::getFullSchema());451 IndexedMemProfData MemProfData = getMemProfDataForTest();452 MemProfData.Records.try_emplace(0x9999, IndexedMR);453 Writer.addMemProfData(MemProfData, Err);454 455 auto Profile = Writer.writeBuffer();456 readProfile(std::move(Profile));457 458 auto RecordOr = Reader->getMemProfRecord(0x9999);459 ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());460 const memprof::MemProfRecord &Record = RecordOr.get();461 462 memprof::IndexedCallstackIdConverter CSIdConv(MemProfData);463 464 const ::llvm::memprof::MemProfRecord WantRecord =465 IndexedMR.toMemProfRecord(CSIdConv);466 ASSERT_EQ(CSIdConv.FrameIdConv.LastUnmappedId, std::nullopt)467 << "could not map frame id: " << *CSIdConv.FrameIdConv.LastUnmappedId;468 ASSERT_EQ(CSIdConv.CSIdConv.LastUnmappedId, std::nullopt)469 << "could not map call stack id: " << *CSIdConv.CSIdConv.LastUnmappedId;470 EXPECT_THAT(WantRecord, EqualsRecord(Record));471}472 473TEST_F(InstrProfTest, test_memprof_v4_partial_schema) {474 const MemInfoBlock MIB = makePartialMIB();475 476 Writer.setMemProfVersionRequested(memprof::Version4);477 Writer.setMemProfFullSchema(false);478 479 ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),480 Succeeded());481 482 const IndexedMemProfRecord IndexedMR = makeRecord(483 /*AllocFrames=*/{0x111, 0x222},484 /*CallSiteFrames=*/{0x333}, MIB, memprof::getHotColdSchema());485 IndexedMemProfData MemProfData = getMemProfDataForTest();486 MemProfData.Records.try_emplace(0x9999, IndexedMR);487 Writer.addMemProfData(MemProfData, Err);488 489 auto Profile = Writer.writeBuffer();490 readProfile(std::move(Profile));491 492 auto RecordOr = Reader->getMemProfRecord(0x9999);493 ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());494 const memprof::MemProfRecord &Record = RecordOr.get();495 496 memprof::IndexedCallstackIdConverter CSIdConv(MemProfData);497 498 const ::llvm::memprof::MemProfRecord WantRecord =499 IndexedMR.toMemProfRecord(CSIdConv);500 ASSERT_EQ(CSIdConv.FrameIdConv.LastUnmappedId, std::nullopt)501 << "could not map frame id: " << *CSIdConv.FrameIdConv.LastUnmappedId;502 ASSERT_EQ(CSIdConv.CSIdConv.LastUnmappedId, std::nullopt)503 << "could not map call stack id: " << *CSIdConv.CSIdConv.LastUnmappedId;504 EXPECT_THAT(WantRecord, EqualsRecord(Record));505}506 507TEST_F(InstrProfTest, test_caller_callee_pairs) {508 const MemInfoBlock MIB = makePartialMIB();509 510 Writer.setMemProfVersionRequested(memprof::Version3);511 Writer.setMemProfFullSchema(false);512 513 ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),514 Succeeded());515 516 // Call Hierarchy517 //518 // Function GUID:0x123519 // Line: 1, Column: 2520 // Function GUID: 0x234521 // Line: 3, Column: 4522 // new(...)523 // Line: 5, Column: 6524 // Function GUID: 0x345525 // Line: 7, Column: 8526 // new(...)527 528 const IndexedMemProfRecord IndexedMR = makeRecord(529 /*AllocFrames=*/{0x111, 0x222},530 /*CallSiteFrames=*/{}, MIB, memprof::getHotColdSchema());531 532 IndexedMemProfData MemProfData;533 MemProfData.Frames.try_emplace(0, 0x123, 1, 2, false);534 MemProfData.Frames.try_emplace(1, 0x234, 3, 4, true);535 MemProfData.Frames.try_emplace(2, 0x123, 5, 6, false);536 MemProfData.Frames.try_emplace(3, 0x345, 7, 8, true);537 MemProfData.CallStacks.try_emplace(538 0x111, std::initializer_list<memprof::FrameId>{1, 0});539 MemProfData.CallStacks.try_emplace(540 0x222, std::initializer_list<memprof::FrameId>{3, 2});541 MemProfData.Records.try_emplace(0x9999, IndexedMR);542 Writer.addMemProfData(MemProfData, Err);543 544 auto Profile = Writer.writeBuffer();545 readProfile(std::move(Profile));546 547 auto Pairs = Reader->getMemProfCallerCalleePairs();548 ASSERT_THAT(Pairs, SizeIs(3));549 550 auto It = Pairs.find(0x123);551 ASSERT_NE(It, Pairs.end());552 EXPECT_THAT(It->second, ElementsAre(Pair(LineLocation(1, 2), 0x234U),553 Pair(LineLocation(5, 6), 0x345U)));554 555 It = Pairs.find(0x234);556 ASSERT_NE(It, Pairs.end());557 EXPECT_THAT(It->second, ElementsAre(Pair(LineLocation(3, 4), 0U)));558 559 It = Pairs.find(0x345);560 ASSERT_NE(It, Pairs.end());561 EXPECT_THAT(It->second, ElementsAre(Pair(LineLocation(7, 8), 0U)));562}563 564TEST_F(InstrProfTest, test_memprof_getrecord_error) {565 ASSERT_THAT_ERROR(Writer.mergeProfileKind(InstrProfKind::MemProf),566 Succeeded());567 568 Writer.setMemProfVersionRequested(memprof::Version3);569 // Generate an empty profile.570 auto Profile = Writer.writeBuffer();571 readProfile(std::move(Profile));572 573 // Missing functions give a unknown_function error.574 auto RecordOr = Reader->getMemProfRecord(0x1111);575 ASSERT_TRUE(576 ErrorEquals(instrprof_error::unknown_function, RecordOr.takeError()));577}578 579TEST_F(InstrProfTest, test_memprof_merge) {580 Writer.addRecord({"func1", 0x1234, {42}}, Err);581 582 InstrProfWriter Writer2;583 Writer2.setMemProfVersionRequested(memprof::Version3);584 ASSERT_THAT_ERROR(Writer2.mergeProfileKind(InstrProfKind::MemProf),585 Succeeded());586 587 const IndexedMemProfRecord IndexedMR = makeRecord(588 /*AllocFrames=*/{0x111, 0x222},589 /*CallSiteFrames=*/{}, makePartialMIB(), memprof::getHotColdSchema());590 591 IndexedMemProfData MemProfData = getMemProfDataForTest();592 MemProfData.Records.try_emplace(0x9999, IndexedMR);593 Writer2.addMemProfData(MemProfData, Err);594 595 ASSERT_THAT_ERROR(Writer.mergeProfileKind(Writer2.getProfileKind()),596 Succeeded());597 Writer.mergeRecordsFromWriter(std::move(Writer2), Err);598 599 Writer.setMemProfVersionRequested(memprof::Version3);600 auto Profile = Writer.writeBuffer();601 readProfile(std::move(Profile));602 603 auto R = Reader->getInstrProfRecord("func1", 0x1234);604 EXPECT_THAT_ERROR(R.takeError(), Succeeded());605 ASSERT_EQ(1U, R->Counts.size());606 ASSERT_EQ(42U, R->Counts[0]);607 608 auto RecordOr = Reader->getMemProfRecord(0x9999);609 ASSERT_THAT_ERROR(RecordOr.takeError(), Succeeded());610 const memprof::MemProfRecord &Record = RecordOr.get();611 612 std::optional<memprof::FrameId> LastUnmappedFrameId;613 614 memprof::IndexedCallstackIdConverter CSIdConv(MemProfData);615 616 const ::llvm::memprof::MemProfRecord WantRecord =617 IndexedMR.toMemProfRecord(CSIdConv);618 ASSERT_EQ(LastUnmappedFrameId, std::nullopt)619 << "could not map frame id: " << *LastUnmappedFrameId;620 EXPECT_THAT(WantRecord, EqualsRecord(Record));621}622 623TEST_F(InstrProfTest, test_irpgo_function_name) {624 LLVMContext Ctx;625 auto M = std::make_unique<Module>("MyModule.cpp", Ctx);626 auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false);627 628 std::vector<std::tuple<StringRef, Function::LinkageTypes, StringRef>> Data;629 Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "ExternalFoo");630 Data.emplace_back("InternalFoo", Function::InternalLinkage,631 "MyModule.cpp;InternalFoo");632 Data.emplace_back("\01-[C dynamicFoo:]", Function::ExternalLinkage,633 "-[C dynamicFoo:]");634 Data.emplace_back("\01-[C internalFoo:]", Function::InternalLinkage,635 "MyModule.cpp;-[C internalFoo:]");636 637 for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data)638 Function::Create(FTy, Linkage, Name, M.get());639 640 for (auto &[Name, Linkage, ExpectedIRPGOFuncName] : Data) {641 auto *F = M->getFunction(Name);642 auto IRPGOFuncName = getIRPGOFuncName(*F);643 EXPECT_EQ(IRPGOFuncName, ExpectedIRPGOFuncName);644 645 auto [Filename, ParsedIRPGOFuncName] = getParsedIRPGOName(IRPGOFuncName);646 StringRef ExpectedParsedIRPGOFuncName = IRPGOFuncName;647 if (ExpectedParsedIRPGOFuncName.consume_front("MyModule.cpp;")) {648 EXPECT_EQ(Filename, "MyModule.cpp");649 } else {650 EXPECT_EQ(Filename, "");651 }652 EXPECT_EQ(ParsedIRPGOFuncName, ExpectedParsedIRPGOFuncName);653 }654}655 656TEST_F(InstrProfTest, test_pgo_function_name) {657 LLVMContext Ctx;658 auto M = std::make_unique<Module>("MyModule.cpp", Ctx);659 auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false);660 661 std::vector<std::tuple<StringRef, Function::LinkageTypes, StringRef>> Data;662 Data.emplace_back("ExternalFoo", Function::ExternalLinkage, "ExternalFoo");663 Data.emplace_back("InternalFoo", Function::InternalLinkage,664 "MyModule.cpp:InternalFoo");665 Data.emplace_back("\01-[C externalFoo:]", Function::ExternalLinkage,666 "-[C externalFoo:]");667 Data.emplace_back("\01-[C internalFoo:]", Function::InternalLinkage,668 "MyModule.cpp:-[C internalFoo:]");669 670 for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data)671 Function::Create(FTy, Linkage, Name, M.get());672 673 for (auto &[Name, Linkage, ExpectedPGOFuncName] : Data) {674 auto *F = M->getFunction(Name);675 EXPECT_EQ(getPGOFuncName(*F), ExpectedPGOFuncName);676 }677}678 679TEST_F(InstrProfTest, test_irpgo_read_deprecated_names) {680 LLVMContext Ctx;681 auto M = std::make_unique<Module>("MyModule.cpp", Ctx);682 auto *FTy = FunctionType::get(Type::getVoidTy(Ctx), /*isVarArg=*/false);683 auto *InternalFooF =684 Function::Create(FTy, Function::InternalLinkage, "InternalFoo", M.get());685 auto *ExternalFooF =686 Function::Create(FTy, Function::ExternalLinkage, "ExternalFoo", M.get());687 688 auto *InternalBarF =689 Function::Create(FTy, Function::InternalLinkage, "InternalBar", M.get());690 auto *ExternalBarF =691 Function::Create(FTy, Function::ExternalLinkage, "ExternalBar", M.get());692 693 Writer.addRecord({getIRPGOFuncName(*InternalFooF), 0x1234, {1}}, Err);694 Writer.addRecord({getIRPGOFuncName(*ExternalFooF), 0x5678, {1}}, Err);695 // Write a record with a deprecated name696 Writer.addRecord({getPGOFuncName(*InternalBarF), 0x1111, {2}}, Err);697 Writer.addRecord({getPGOFuncName(*ExternalBarF), 0x2222, {2}}, Err);698 699 auto Profile = Writer.writeBuffer();700 readProfile(std::move(Profile));701 702 EXPECT_THAT_EXPECTED(703 Reader->getInstrProfRecord(getIRPGOFuncName(*InternalFooF), 0x1234,704 getPGOFuncName(*InternalFooF)),705 Succeeded());706 EXPECT_THAT_EXPECTED(707 Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalFooF), 0x5678,708 getPGOFuncName(*ExternalFooF)),709 Succeeded());710 // Ensure we can still read this old record name711 EXPECT_THAT_EXPECTED(712 Reader->getInstrProfRecord(getIRPGOFuncName(*InternalBarF), 0x1111,713 getPGOFuncName(*InternalBarF)),714 Succeeded());715 EXPECT_THAT_EXPECTED(716 Reader->getInstrProfRecord(getIRPGOFuncName(*ExternalBarF), 0x2222,717 getPGOFuncName(*ExternalBarF)),718 Succeeded());719}720 721// callee1 to callee6 are from vtable1 to vtable6 respectively.722static const char callee1[] = "callee1";723static const char callee2[] = "callee2";724static const char callee3[] = "callee3";725static const char callee4[] = "callee4";726static const char callee5[] = "callee5";727static const char callee6[] = "callee6";728// callee7 and callee8 are not from any vtables.729static const char callee7[] = "callee7";730static const char callee8[] = "callee8";731// 'callee' is primarily used to create multiple-element vtables.732static const char callee[] = "callee";733static const uint64_t vtable1[] = {uint64_t(callee), uint64_t(callee1)};734static const uint64_t vtable2[] = {uint64_t(callee2), uint64_t(callee)};735static const uint64_t vtable3[] = {736 uint64_t(callee),737 uint64_t(callee3),738};739static const uint64_t vtable4[] = {uint64_t(callee4), uint64_t(callee)};740static const uint64_t vtable5[] = {uint64_t(callee5), uint64_t(callee)};741static const uint64_t vtable6[] = {uint64_t(callee6), uint64_t(callee)};742 743// Returns the address of callee with a numbered suffix in vtable.744static uint64_t getCalleeAddress(const uint64_t *vtableAddr) {745 uint64_t CalleeAddr;746 // Callee with a numbered suffix is the 2nd element in vtable1 and vtable3,747 // and the 1st element in the rest of vtables.748 if (vtableAddr == vtable1 || vtableAddr == vtable3)749 CalleeAddr = uint64_t(vtableAddr) + 8;750 else751 CalleeAddr = uint64_t(vtableAddr);752 return CalleeAddr;753}754 755TEST_P(InstrProfReaderWriterTest, icall_and_vtable_data_read_write) {756 NamedInstrProfRecord Record1("caller", 0x1234, {1, 2});757 758 // 4 indirect call value sites.759 {760 Record1.reserveSites(IPVK_IndirectCallTarget, 4);761 InstrProfValueData VD0[] = {762 {(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}, {(uint64_t)callee3, 3}};763 Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, nullptr);764 // No value profile data at the second site.765 Record1.addValueData(IPVK_IndirectCallTarget, 1, {}, nullptr);766 InstrProfValueData VD2[] = {{(uint64_t)callee1, 1}, {(uint64_t)callee2, 2}};767 Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, nullptr);768 InstrProfValueData VD3[] = {{(uint64_t)callee7, 1}, {(uint64_t)callee8, 2}};769 Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, nullptr);770 }771 772 // 2 vtable value sites.773 {774 InstrProfValueData VD0[] = {775 {getCalleeAddress(vtable1), 1},776 {getCalleeAddress(vtable2), 2},777 {getCalleeAddress(vtable3), 3},778 };779 InstrProfValueData VD2[] = {780 {getCalleeAddress(vtable1), 1},781 {getCalleeAddress(vtable2), 2},782 };783 Record1.addValueData(IPVK_VTableTarget, 0, VD0, nullptr);784 Record1.addValueData(IPVK_VTableTarget, 1, VD2, nullptr);785 }786 787 Writer.addRecord(std::move(Record1), getProfWeight(), Err);788 Writer.addRecord({"callee1", 0x1235, {3, 4}}, Err);789 Writer.addRecord({"callee2", 0x1235, {3, 4}}, Err);790 Writer.addRecord({"callee3", 0x1235, {3, 4}}, Err);791 Writer.addRecord({"callee7", 0x1235, {3, 4}}, Err);792 Writer.addRecord({"callee8", 0x1235, {3, 4}}, Err);793 794 // Set writer value prof data endianness.795 Writer.setValueProfDataEndianness(getEndianness());796 797 auto Profile = Writer.writeBuffer();798 readProfile(std::move(Profile));799 800 // Set reader value prof data endianness.801 Reader->setValueProfDataEndianness(getEndianness());802 803 auto R = Reader->getInstrProfRecord("caller", 0x1234);804 ASSERT_THAT_ERROR(R.takeError(), Succeeded());805 806 // Test the number of instrumented indirect call sites and the number of807 // profiled values at each site.808 ASSERT_EQ(4U, R->getNumValueSites(IPVK_IndirectCallTarget));809 810 // Test the number of instrumented vtable sites and the number of profiled811 // values at each site.812 ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 2U);813 814 // First indirect site.815 {816 auto VD = R->getValueArrayForSite(IPVK_IndirectCallTarget, 0);817 ASSERT_THAT(VD, SizeIs(3));818 819 EXPECT_EQ(VD[0].Count, 3U * getProfWeight());820 EXPECT_EQ(VD[1].Count, 2U * getProfWeight());821 EXPECT_EQ(VD[2].Count, 1U * getProfWeight());822 823 EXPECT_STREQ((const char *)VD[0].Value, "callee3");824 EXPECT_STREQ((const char *)VD[1].Value, "callee2");825 EXPECT_STREQ((const char *)VD[2].Value, "callee1");826 }827 828 EXPECT_THAT(R->getValueArrayForSite(IPVK_IndirectCallTarget, 1), SizeIs(0));829 EXPECT_THAT(R->getValueArrayForSite(IPVK_IndirectCallTarget, 2), SizeIs(2));830 EXPECT_THAT(R->getValueArrayForSite(IPVK_IndirectCallTarget, 3), SizeIs(2));831 832 // First vtable site.833 {834 auto VD = R->getValueArrayForSite(IPVK_VTableTarget, 0);835 ASSERT_THAT(VD, SizeIs(3));836 837 EXPECT_EQ(VD[0].Count, 3U * getProfWeight());838 EXPECT_EQ(VD[1].Count, 2U * getProfWeight());839 EXPECT_EQ(VD[2].Count, 1U * getProfWeight());840 841 EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable3));842 EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable2));843 EXPECT_EQ(VD[2].Value, getCalleeAddress(vtable1));844 }845 846 // Second vtable site.847 {848 auto VD = R->getValueArrayForSite(IPVK_VTableTarget, 1);849 ASSERT_THAT(VD, SizeIs(2));850 851 EXPECT_EQ(VD[0].Count, 2U * getProfWeight());852 EXPECT_EQ(VD[1].Count, 1U * getProfWeight());853 854 EXPECT_EQ(VD[0].Value, getCalleeAddress(vtable2));855 EXPECT_EQ(VD[1].Value, getCalleeAddress(vtable1));856 }857}858 859INSTANTIATE_TEST_SUITE_P(860 WeightAndEndiannessTest, InstrProfReaderWriterTest,861 ::testing::Combine(862 ::testing::Bool(), /* Sparse */863 ::testing::Values(1U, 10U), /* ProfWeight */864 ::testing::Values(llvm::endianness::big,865 llvm::endianness::little) /* Endianness */866 ));867 868TEST_P(MaybeSparseInstrProfTest, annotate_vp_data) {869 NamedInstrProfRecord Record("caller", 0x1234, {1, 2});870 Record.reserveSites(IPVK_IndirectCallTarget, 1);871 InstrProfValueData VD0[] = {{1000, 1}, {2000, 2}, {3000, 3}, {5000, 5},872 {4000, 4}, {6000, 6}};873 Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, nullptr);874 Writer.addRecord(std::move(Record), Err);875 auto Profile = Writer.writeBuffer();876 readProfile(std::move(Profile));877 auto R = Reader->getInstrProfRecord("caller", 0x1234);878 EXPECT_THAT_ERROR(R.takeError(), Succeeded());879 880 LLVMContext Ctx;881 std::unique_ptr<Module> M(new Module("MyModule", Ctx));882 FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),883 /*isVarArg=*/false);884 Function *F =885 Function::Create(FTy, Function::ExternalLinkage, "caller", M.get());886 BasicBlock *BB = BasicBlock::Create(Ctx, "", F);887 888 IRBuilder<> Builder(BB);889 BasicBlock *TBB = BasicBlock::Create(Ctx, "", F);890 BasicBlock *FBB = BasicBlock::Create(Ctx, "", F);891 892 // Use branch instruction to annotate with value profile data for simplicity893 Instruction *Inst = Builder.CreateCondBr(Builder.getTrue(), TBB, FBB);894 Instruction *Inst2 = Builder.CreateCondBr(Builder.getTrue(), TBB, FBB);895 annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0);896 897 uint64_t T;898 auto ValueData =899 getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5, T);900 ASSERT_THAT(ValueData, SizeIs(3));901 ASSERT_EQ(21U, T);902 // The result should be sorted already:903 ASSERT_EQ(6000U, ValueData[0].Value);904 ASSERT_EQ(6U, ValueData[0].Count);905 ASSERT_EQ(5000U, ValueData[1].Value);906 ASSERT_EQ(5U, ValueData[1].Count);907 ASSERT_EQ(4000U, ValueData[2].Value);908 ASSERT_EQ(4U, ValueData[2].Count);909 ValueData = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 1, T);910 ASSERT_THAT(ValueData, SizeIs(1));911 ASSERT_EQ(21U, T);912 913 ValueData = getValueProfDataFromInst(*Inst2, IPVK_IndirectCallTarget, 5, T);914 ASSERT_THAT(ValueData, SizeIs(0));915 916 // Remove the MD_prof metadata917 Inst->setMetadata(LLVMContext::MD_prof, nullptr);918 // Annotate 5 records this time.919 annotateValueSite(*M, *Inst, R.get(), IPVK_IndirectCallTarget, 0, 5);920 ValueData = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5, T);921 ASSERT_THAT(ValueData, SizeIs(5));922 ASSERT_EQ(21U, T);923 ASSERT_EQ(6000U, ValueData[0].Value);924 ASSERT_EQ(6U, ValueData[0].Count);925 ASSERT_EQ(5000U, ValueData[1].Value);926 ASSERT_EQ(5U, ValueData[1].Count);927 ASSERT_EQ(4000U, ValueData[2].Value);928 ASSERT_EQ(4U, ValueData[2].Count);929 ASSERT_EQ(3000U, ValueData[3].Value);930 ASSERT_EQ(3U, ValueData[3].Count);931 ASSERT_EQ(2000U, ValueData[4].Value);932 ASSERT_EQ(2U, ValueData[4].Count);933 934 // Remove the MD_prof metadata935 Inst->setMetadata(LLVMContext::MD_prof, nullptr);936 // Annotate with 4 records.937 InstrProfValueData VD0Sorted[] = {{1000, 6}, {2000, 5}, {3000, 4}, {4000, 3},938 {5000, 2}, {6000, 1}};939 annotateValueSite(*M, *Inst, ArrayRef(VD0Sorted).slice(2), 10,940 IPVK_IndirectCallTarget, 5);941 ValueData = getValueProfDataFromInst(*Inst, IPVK_IndirectCallTarget, 5, T);942 ASSERT_THAT(ValueData, SizeIs(4));943 ASSERT_EQ(10U, T);944 ASSERT_EQ(3000U, ValueData[0].Value);945 ASSERT_EQ(4U, ValueData[0].Count);946 ASSERT_EQ(4000U, ValueData[1].Value);947 ASSERT_EQ(3U, ValueData[1].Count);948 ASSERT_EQ(5000U, ValueData[2].Value);949 ASSERT_EQ(2U, ValueData[2].Count);950 ASSERT_EQ(6000U, ValueData[3].Value);951 ASSERT_EQ(1U, ValueData[3].Count);952}953 954TEST_P(MaybeSparseInstrProfTest, icall_and_vtable_data_merge) {955 static const char caller[] = "caller";956 NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});957 NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});958 959 // 5 value sites for indirect calls.960 {961 Record11.reserveSites(IPVK_IndirectCallTarget, 5);962 InstrProfValueData VD0[] = {{uint64_t(callee1), 1},963 {uint64_t(callee2), 2},964 {uint64_t(callee3), 3},965 {uint64_t(callee4), 4}};966 Record11.addValueData(IPVK_IndirectCallTarget, 0, VD0, nullptr);967 968 // No value profile data at the second site.969 Record11.addValueData(IPVK_IndirectCallTarget, 1, {}, nullptr);970 971 InstrProfValueData VD2[] = {972 {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};973 Record11.addValueData(IPVK_IndirectCallTarget, 2, VD2, nullptr);974 975 InstrProfValueData VD3[] = {{uint64_t(callee7), 1}, {uint64_t(callee8), 2}};976 Record11.addValueData(IPVK_IndirectCallTarget, 3, VD3, nullptr);977 978 InstrProfValueData VD4[] = {979 {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};980 Record11.addValueData(IPVK_IndirectCallTarget, 4, VD4, nullptr);981 }982 // 3 value sites for vtables.983 {984 Record11.reserveSites(IPVK_VTableTarget, 3);985 InstrProfValueData VD0[] = {{getCalleeAddress(vtable1), 1},986 {getCalleeAddress(vtable2), 2},987 {getCalleeAddress(vtable3), 3},988 {getCalleeAddress(vtable4), 4}};989 Record11.addValueData(IPVK_VTableTarget, 0, VD0, nullptr);990 991 InstrProfValueData VD2[] = {{getCalleeAddress(vtable1), 1},992 {getCalleeAddress(vtable2), 2},993 {getCalleeAddress(vtable3), 3}};994 Record11.addValueData(IPVK_VTableTarget, 1, VD2, nullptr);995 996 InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1},997 {getCalleeAddress(vtable2), 2},998 {getCalleeAddress(vtable3), 3}};999 Record11.addValueData(IPVK_VTableTarget, 2, VD4, nullptr);1000 }1001 1002 // A different record for the same caller.1003 Record12.reserveSites(IPVK_IndirectCallTarget, 5);1004 InstrProfValueData VD02[] = {{uint64_t(callee2), 5}, {uint64_t(callee3), 3}};1005 Record12.addValueData(IPVK_IndirectCallTarget, 0, VD02, nullptr);1006 1007 // No value profile data at the second site.1008 Record12.addValueData(IPVK_IndirectCallTarget, 1, {}, nullptr);1009 1010 InstrProfValueData VD22[] = {1011 {uint64_t(callee2), 1}, {uint64_t(callee3), 3}, {uint64_t(callee4), 4}};1012 Record12.addValueData(IPVK_IndirectCallTarget, 2, VD22, nullptr);1013 1014 Record12.addValueData(IPVK_IndirectCallTarget, 3, {}, nullptr);1015 1016 InstrProfValueData VD42[] = {1017 {uint64_t(callee1), 1}, {uint64_t(callee2), 2}, {uint64_t(callee3), 3}};1018 Record12.addValueData(IPVK_IndirectCallTarget, 4, VD42, nullptr);1019 1020 // 3 value sites for vtables.1021 {1022 Record12.reserveSites(IPVK_VTableTarget, 3);1023 InstrProfValueData VD0[] = {{getCalleeAddress(vtable2), 5},1024 {getCalleeAddress(vtable3), 3}};1025 Record12.addValueData(IPVK_VTableTarget, 0, VD0, nullptr);1026 1027 InstrProfValueData VD2[] = {{getCalleeAddress(vtable2), 1},1028 {getCalleeAddress(vtable3), 3},1029 {getCalleeAddress(vtable4), 4}};1030 Record12.addValueData(IPVK_VTableTarget, 1, VD2, nullptr);1031 1032 InstrProfValueData VD4[] = {{getCalleeAddress(vtable1), 1},1033 {getCalleeAddress(vtable2), 2},1034 {getCalleeAddress(vtable3), 3}};1035 Record12.addValueData(IPVK_VTableTarget, 2, VD4, nullptr);1036 }1037 1038 Writer.addRecord(std::move(Record11), Err);1039 // Merge profile data.1040 Writer.addRecord(std::move(Record12), Err);1041 1042 Writer.addRecord({callee1, 0x1235, {3, 4}}, Err);1043 Writer.addRecord({callee2, 0x1235, {3, 4}}, Err);1044 Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);1045 Writer.addRecord({callee3, 0x1235, {3, 4}}, Err);1046 Writer.addRecord({callee4, 0x1235, {3, 5}}, Err);1047 Writer.addRecord({callee7, 0x1235, {3, 5}}, Err);1048 Writer.addRecord({callee8, 0x1235, {3, 5}}, Err);1049 auto Profile = Writer.writeBuffer();1050 readProfile(std::move(Profile));1051 1052 // Test the number of instrumented value sites and the number of profiled1053 // values for each site.1054 auto R = Reader->getInstrProfRecord("caller", 0x1234);1055 EXPECT_THAT_ERROR(R.takeError(), Succeeded());1056 // For indirect calls.1057 ASSERT_EQ(5U, R->getNumValueSites(IPVK_IndirectCallTarget));1058 // For vtables.1059 ASSERT_EQ(R->getNumValueSites(IPVK_VTableTarget), 3U);1060 1061 // Test the merged values for indirect calls.1062 {1063 auto VD = R->getValueArrayForSite(IPVK_IndirectCallTarget, 0);1064 ASSERT_THAT(VD, SizeIs(4));1065 EXPECT_STREQ((const char *)VD[0].Value, "callee2");1066 EXPECT_EQ(VD[0].Count, 7U);1067 EXPECT_STREQ((const char *)VD[1].Value, "callee3");1068 EXPECT_EQ(VD[1].Count, 6U);1069 EXPECT_STREQ((const char *)VD[2].Value, "callee4");1070 EXPECT_EQ(VD[2].Count, 4U);1071 EXPECT_STREQ((const char *)VD[3].Value, "callee1");1072 EXPECT_EQ(VD[3].Count, 1U);1073 1074 ASSERT_THAT(R->getValueArrayForSite(IPVK_IndirectCallTarget, 1), SizeIs(0));1075 1076 auto VD_2 = R->getValueArrayForSite(IPVK_IndirectCallTarget, 2);1077 ASSERT_THAT(VD_2, SizeIs(4));1078 EXPECT_STREQ((const char *)VD_2[0].Value, "callee3");1079 EXPECT_EQ(VD_2[0].Count, 6U);1080 EXPECT_STREQ((const char *)VD_2[1].Value, "callee4");1081 EXPECT_EQ(VD_2[1].Count, 4U);1082 EXPECT_STREQ((const char *)VD_2[2].Value, "callee2");1083 EXPECT_EQ(VD_2[2].Count, 3U);1084 EXPECT_STREQ((const char *)VD_2[3].Value, "callee1");1085 EXPECT_EQ(VD_2[3].Count, 1U);1086 1087 auto VD_3 = R->getValueArrayForSite(IPVK_IndirectCallTarget, 3);1088 ASSERT_THAT(VD_3, SizeIs(2));1089 EXPECT_STREQ((const char *)VD_3[0].Value, "callee8");1090 EXPECT_EQ(VD_3[0].Count, 2U);1091 EXPECT_STREQ((const char *)VD_3[1].Value, "callee7");1092 EXPECT_EQ(VD_3[1].Count, 1U);1093 1094 auto VD_4 = R->getValueArrayForSite(IPVK_IndirectCallTarget, 4);1095 ASSERT_THAT(VD_4, SizeIs(3));1096 EXPECT_STREQ((const char *)VD_4[0].Value, "callee3");1097 EXPECT_EQ(VD_4[0].Count, 6U);1098 EXPECT_STREQ((const char *)VD_4[1].Value, "callee2");1099 EXPECT_EQ(VD_4[1].Count, 4U);1100 EXPECT_STREQ((const char *)VD_4[2].Value, "callee1");1101 EXPECT_EQ(VD_4[2].Count, 2U);1102 }1103 1104 // Test the merged values for vtables1105 {1106 auto VD0 = R->getValueArrayForSite(IPVK_VTableTarget, 0);1107 ASSERT_THAT(VD0, SizeIs(4));1108 EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2));1109 EXPECT_EQ(VD0[0].Count, 7U);1110 EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3));1111 EXPECT_EQ(VD0[1].Count, 6U);1112 EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable4));1113 EXPECT_EQ(VD0[2].Count, 4U);1114 EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable1));1115 EXPECT_EQ(VD0[3].Count, 1U);1116 1117 auto VD1 = R->getValueArrayForSite(IPVK_VTableTarget, 1);1118 ASSERT_THAT(VD1, SizeIs(4));1119 EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable3));1120 EXPECT_EQ(VD1[0].Count, 6U);1121 EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable4));1122 EXPECT_EQ(VD1[1].Count, 4U);1123 EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable2));1124 EXPECT_EQ(VD1[2].Count, 3U);1125 EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable1));1126 EXPECT_EQ(VD1[3].Count, 1U);1127 1128 auto VD2 = R->getValueArrayForSite(IPVK_VTableTarget, 2);1129 ASSERT_THAT(VD2, SizeIs(3));1130 EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable3));1131 EXPECT_EQ(VD2[0].Count, 6U);1132 EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable2));1133 EXPECT_EQ(VD2[1].Count, 4U);1134 EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable1));1135 EXPECT_EQ(VD2[2].Count, 2U);1136 }1137}1138 1139struct ValueProfileMergeEdgeCaseTest1140 : public InstrProfTest,1141 public ::testing::WithParamInterface<std::tuple<bool, uint32_t>> {1142 void SetUp() override { Writer.setOutputSparse(std::get<0>(GetParam())); }1143 1144 uint32_t getValueProfileKind() const { return std::get<1>(GetParam()); }1145};1146 1147TEST_P(ValueProfileMergeEdgeCaseTest, value_profile_data_merge_saturation) {1148 const uint32_t ValueKind = getValueProfileKind();1149 static const char bar[] = "bar";1150 const uint64_t ProfiledValue = 0x5678;1151 1152 const uint64_t MaxValCount = std::numeric_limits<uint64_t>::max();1153 const uint64_t MaxEdgeCount = getInstrMaxCountValue();1154 1155 instrprof_error Result;1156 auto Err = [&](Error E) {1157 Result = std::get<0>(InstrProfError::take(std::move(E)));1158 };1159 Result = instrprof_error::success;1160 Writer.addRecord({"foo", 0x1234, {1}}, Err);1161 ASSERT_EQ(Result, instrprof_error::success);1162 1163 // Verify counter overflow.1164 Result = instrprof_error::success;1165 Writer.addRecord({"foo", 0x1234, {MaxEdgeCount}}, Err);1166 ASSERT_EQ(Result, instrprof_error::counter_overflow);1167 1168 Result = instrprof_error::success;1169 Writer.addRecord({bar, 0x9012, {8}}, Err);1170 ASSERT_EQ(Result, instrprof_error::success);1171 1172 NamedInstrProfRecord Record4("baz", 0x5678, {3, 4});1173 Record4.reserveSites(ValueKind, 1);1174 InstrProfValueData VD4[] = {{ProfiledValue, 1}};1175 Record4.addValueData(ValueKind, 0, VD4, nullptr);1176 Result = instrprof_error::success;1177 Writer.addRecord(std::move(Record4), Err);1178 ASSERT_EQ(Result, instrprof_error::success);1179 1180 // Verify value data counter overflow.1181 NamedInstrProfRecord Record5("baz", 0x5678, {5, 6});1182 Record5.reserveSites(ValueKind, 1);1183 InstrProfValueData VD5[] = {{ProfiledValue, MaxValCount}};1184 Record5.addValueData(ValueKind, 0, VD5, nullptr);1185 Result = instrprof_error::success;1186 Writer.addRecord(std::move(Record5), Err);1187 ASSERT_EQ(Result, instrprof_error::counter_overflow);1188 1189 auto Profile = Writer.writeBuffer();1190 readProfile(std::move(Profile));1191 1192 // Verify saturation of counts.1193 auto ReadRecord1 = Reader->getInstrProfRecord("foo", 0x1234);1194 ASSERT_THAT_ERROR(ReadRecord1.takeError(), Succeeded());1195 EXPECT_EQ(MaxEdgeCount, ReadRecord1->Counts[0]);1196 1197 auto ReadRecord2 = Reader->getInstrProfRecord("baz", 0x5678);1198 ASSERT_TRUE(bool(ReadRecord2));1199 ASSERT_EQ(1U, ReadRecord2->getNumValueSites(ValueKind));1200 auto VD = ReadRecord2->getValueArrayForSite(ValueKind, 0);1201 EXPECT_EQ(ProfiledValue, VD[0].Value);1202 EXPECT_EQ(MaxValCount, VD[0].Count);1203}1204 1205// This test tests that when there are too many values for a given site, the1206// merged results are properly truncated.1207TEST_P(ValueProfileMergeEdgeCaseTest, value_profile_data_merge_site_trunc) {1208 const uint32_t ValueKind = getValueProfileKind();1209 static const char caller[] = "caller";1210 1211 NamedInstrProfRecord Record11(caller, 0x1234, {1, 2});1212 NamedInstrProfRecord Record12(caller, 0x1234, {1, 2});1213 1214 // 2 value sites.1215 Record11.reserveSites(ValueKind, 2);1216 InstrProfValueData VD0[255];1217 for (int I = 0; I < 255; I++) {1218 VD0[I].Value = 2 * I;1219 VD0[I].Count = 2 * I + 1000;1220 }1221 1222 Record11.addValueData(ValueKind, 0, VD0, nullptr);1223 Record11.addValueData(ValueKind, 1, {}, nullptr);1224 1225 Record12.reserveSites(ValueKind, 2);1226 InstrProfValueData VD1[255];1227 for (int I = 0; I < 255; I++) {1228 VD1[I].Value = 2 * I + 1;1229 VD1[I].Count = 2 * I + 1001;1230 }1231 1232 Record12.addValueData(ValueKind, 0, VD1, nullptr);1233 Record12.addValueData(ValueKind, 1, {}, nullptr);1234 1235 Writer.addRecord(std::move(Record11), Err);1236 // Merge profile data.1237 Writer.addRecord(std::move(Record12), Err);1238 1239 auto Profile = Writer.writeBuffer();1240 readProfile(std::move(Profile));1241 1242 auto R = Reader->getInstrProfRecord("caller", 0x1234);1243 ASSERT_THAT_ERROR(R.takeError(), Succeeded());1244 ASSERT_EQ(2U, R->getNumValueSites(ValueKind));1245 auto VD = R->getValueArrayForSite(ValueKind, 0);1246 EXPECT_THAT(VD, SizeIs(255));1247 for (unsigned I = 0; I < 255; I++) {1248 EXPECT_EQ(VD[I].Value, 509 - I);1249 EXPECT_EQ(VD[I].Count, 1509 - I);1250 }1251}1252 1253INSTANTIATE_TEST_SUITE_P(1254 EdgeCaseTest, ValueProfileMergeEdgeCaseTest,1255 ::testing::Combine(::testing::Bool(), /* Sparse */1256 ::testing::Values(IPVK_IndirectCallTarget,1257 IPVK_MemOPSize,1258 IPVK_VTableTarget) /* ValueKind */1259 ));1260 1261static void addValueProfData(InstrProfRecord &Record) {1262 // Add test data for indirect calls.1263 {1264 Record.reserveSites(IPVK_IndirectCallTarget, 6);1265 InstrProfValueData VD0[] = {{uint64_t(callee1), 400},1266 {uint64_t(callee2), 1000},1267 {uint64_t(callee3), 500},1268 {uint64_t(callee4), 300},1269 {uint64_t(callee5), 100}};1270 Record.addValueData(IPVK_IndirectCallTarget, 0, VD0, nullptr);1271 InstrProfValueData VD1[] = {{uint64_t(callee5), 800},1272 {uint64_t(callee3), 1000},1273 {uint64_t(callee2), 2500},1274 {uint64_t(callee1), 1300}};1275 Record.addValueData(IPVK_IndirectCallTarget, 1, VD1, nullptr);1276 InstrProfValueData VD2[] = {{uint64_t(callee6), 800},1277 {uint64_t(callee3), 1000},1278 {uint64_t(callee4), 5500}};1279 Record.addValueData(IPVK_IndirectCallTarget, 2, VD2, nullptr);1280 InstrProfValueData VD3[] = {{uint64_t(callee2), 1800},1281 {uint64_t(callee3), 2000}};1282 Record.addValueData(IPVK_IndirectCallTarget, 3, VD3, nullptr);1283 Record.addValueData(IPVK_IndirectCallTarget, 4, {}, nullptr);1284 InstrProfValueData VD5[] = {{uint64_t(callee7), 1234},1285 {uint64_t(callee8), 5678}};1286 Record.addValueData(IPVK_IndirectCallTarget, 5, VD5, nullptr);1287 }1288 1289 // Add test data for vtables1290 {1291 Record.reserveSites(IPVK_VTableTarget, 4);1292 InstrProfValueData VD0[] = {1293 {getCalleeAddress(vtable1), 400}, {getCalleeAddress(vtable2), 1000},1294 {getCalleeAddress(vtable3), 500}, {getCalleeAddress(vtable4), 300},1295 {getCalleeAddress(vtable5), 100},1296 };1297 InstrProfValueData VD1[] = {{getCalleeAddress(vtable5), 800},1298 {getCalleeAddress(vtable3), 1000},1299 {getCalleeAddress(vtable2), 2500},1300 {getCalleeAddress(vtable1), 1300}};1301 InstrProfValueData VD2[] = {1302 {getCalleeAddress(vtable6), 800},1303 {getCalleeAddress(vtable3), 1000},1304 {getCalleeAddress(vtable4), 5500},1305 };1306 InstrProfValueData VD3[] = {{getCalleeAddress(vtable2), 1800},1307 {getCalleeAddress(vtable3), 2000}};1308 Record.addValueData(IPVK_VTableTarget, 0, VD0, nullptr);1309 Record.addValueData(IPVK_VTableTarget, 1, VD1, nullptr);1310 Record.addValueData(IPVK_VTableTarget, 2, VD2, nullptr);1311 Record.addValueData(IPVK_VTableTarget, 3, VD3, nullptr);1312 }1313}1314 1315TEST(ValueProfileReadWriteTest, value_prof_data_read_write) {1316 InstrProfRecord SrcRecord({1ULL << 31, 2});1317 addValueProfData(SrcRecord);1318 std::unique_ptr<ValueProfData> VPData =1319 ValueProfData::serializeFrom(SrcRecord);1320 1321 InstrProfRecord Record({1ULL << 31, 2});1322 VPData->deserializeTo(Record, nullptr);1323 1324 // Now read data from Record and sanity check the data1325 ASSERT_EQ(6U, Record.getNumValueSites(IPVK_IndirectCallTarget));1326 1327 auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {1328 return VD1.Count > VD2.Count;1329 };1330 1331 SmallVector<InstrProfValueData> VD_0(1332 Record.getValueArrayForSite(IPVK_IndirectCallTarget, 0));1333 ASSERT_THAT(VD_0, SizeIs(5));1334 llvm::sort(VD_0, Cmp);1335 EXPECT_STREQ((const char *)VD_0[0].Value, "callee2");1336 EXPECT_EQ(1000U, VD_0[0].Count);1337 EXPECT_STREQ((const char *)VD_0[1].Value, "callee3");1338 EXPECT_EQ(500U, VD_0[1].Count);1339 EXPECT_STREQ((const char *)VD_0[2].Value, "callee1");1340 EXPECT_EQ(400U, VD_0[2].Count);1341 EXPECT_STREQ((const char *)VD_0[3].Value, "callee4");1342 EXPECT_EQ(300U, VD_0[3].Count);1343 EXPECT_STREQ((const char *)VD_0[4].Value, "callee5");1344 EXPECT_EQ(100U, VD_0[4].Count);1345 1346 SmallVector<InstrProfValueData> VD_1(1347 Record.getValueArrayForSite(IPVK_IndirectCallTarget, 1));1348 ASSERT_THAT(VD_1, SizeIs(4));1349 llvm::sort(VD_1, Cmp);1350 EXPECT_STREQ((const char *)VD_1[0].Value, "callee2");1351 EXPECT_EQ(VD_1[0].Count, 2500U);1352 EXPECT_STREQ((const char *)VD_1[1].Value, "callee1");1353 EXPECT_EQ(VD_1[1].Count, 1300U);1354 EXPECT_STREQ((const char *)VD_1[2].Value, "callee3");1355 EXPECT_EQ(VD_1[2].Count, 1000U);1356 EXPECT_STREQ((const char *)VD_1[3].Value, "callee5");1357 EXPECT_EQ(VD_1[3].Count, 800U);1358 1359 SmallVector<InstrProfValueData> VD_2(1360 Record.getValueArrayForSite(IPVK_IndirectCallTarget, 2));1361 ASSERT_THAT(VD_2, SizeIs(3));1362 llvm::sort(VD_2, Cmp);1363 EXPECT_STREQ((const char *)VD_2[0].Value, "callee4");1364 EXPECT_EQ(VD_2[0].Count, 5500U);1365 EXPECT_STREQ((const char *)VD_2[1].Value, "callee3");1366 EXPECT_EQ(VD_2[1].Count, 1000U);1367 EXPECT_STREQ((const char *)VD_2[2].Value, "callee6");1368 EXPECT_EQ(VD_2[2].Count, 800U);1369 1370 SmallVector<InstrProfValueData> VD_3(1371 Record.getValueArrayForSite(IPVK_IndirectCallTarget, 3));1372 ASSERT_THAT(VD_3, SizeIs(2));1373 llvm::sort(VD_3, Cmp);1374 EXPECT_STREQ((const char *)VD_3[0].Value, "callee3");1375 EXPECT_EQ(VD_3[0].Count, 2000U);1376 EXPECT_STREQ((const char *)VD_3[1].Value, "callee2");1377 EXPECT_EQ(VD_3[1].Count, 1800U);1378 1379 ASSERT_THAT(Record.getValueArrayForSite(IPVK_IndirectCallTarget, 4),1380 SizeIs(0));1381 ASSERT_THAT(Record.getValueArrayForSite(IPVK_IndirectCallTarget, 5),1382 SizeIs(2));1383 1384 ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U);1385 1386 SmallVector<InstrProfValueData> VD0(1387 Record.getValueArrayForSite(IPVK_VTableTarget, 0));1388 ASSERT_THAT(VD0, SizeIs(5));1389 llvm::sort(VD0, Cmp);1390 EXPECT_EQ(VD0[0].Value, getCalleeAddress(vtable2));1391 EXPECT_EQ(VD0[0].Count, 1000U);1392 EXPECT_EQ(VD0[1].Value, getCalleeAddress(vtable3));1393 EXPECT_EQ(VD0[1].Count, 500U);1394 EXPECT_EQ(VD0[2].Value, getCalleeAddress(vtable1));1395 EXPECT_EQ(VD0[2].Count, 400U);1396 EXPECT_EQ(VD0[3].Value, getCalleeAddress(vtable4));1397 EXPECT_EQ(VD0[3].Count, 300U);1398 EXPECT_EQ(VD0[4].Value, getCalleeAddress(vtable5));1399 EXPECT_EQ(VD0[4].Count, 100U);1400 1401 SmallVector<InstrProfValueData> VD1(1402 Record.getValueArrayForSite(IPVK_VTableTarget, 1));1403 ASSERT_THAT(VD1, SizeIs(4));1404 llvm::sort(VD1, Cmp);1405 EXPECT_EQ(VD1[0].Value, getCalleeAddress(vtable2));1406 EXPECT_EQ(VD1[0].Count, 2500U);1407 EXPECT_EQ(VD1[1].Value, getCalleeAddress(vtable1));1408 EXPECT_EQ(VD1[1].Count, 1300U);1409 EXPECT_EQ(VD1[2].Value, getCalleeAddress(vtable3));1410 EXPECT_EQ(VD1[2].Count, 1000U);1411 EXPECT_EQ(VD1[3].Value, getCalleeAddress(vtable5));1412 EXPECT_EQ(VD1[3].Count, 800U);1413 1414 SmallVector<InstrProfValueData> VD2(1415 Record.getValueArrayForSite(IPVK_VTableTarget, 2));1416 ASSERT_THAT(VD2, SizeIs(3));1417 llvm::sort(VD2, Cmp);1418 EXPECT_EQ(VD2[0].Value, getCalleeAddress(vtable4));1419 EXPECT_EQ(VD2[0].Count, 5500U);1420 EXPECT_EQ(VD2[1].Value, getCalleeAddress(vtable3));1421 EXPECT_EQ(VD2[1].Count, 1000U);1422 EXPECT_EQ(VD2[2].Value, getCalleeAddress(vtable6));1423 EXPECT_EQ(VD2[2].Count, 800U);1424 1425 SmallVector<InstrProfValueData> VD3(1426 Record.getValueArrayForSite(IPVK_VTableTarget, 3));1427 ASSERT_THAT(VD3, SizeIs(2));1428 llvm::sort(VD3, Cmp);1429 EXPECT_EQ(VD3[0].Value, getCalleeAddress(vtable3));1430 EXPECT_EQ(VD3[0].Count, 2000U);1431 EXPECT_EQ(VD3[1].Value, getCalleeAddress(vtable2));1432 EXPECT_EQ(VD3[1].Count, 1800U);1433}1434 1435TEST(ValueProfileReadWriteTest, symtab_mapping) {1436 NamedInstrProfRecord SrcRecord("caller", 0x1234, {1ULL << 31, 2});1437 addValueProfData(SrcRecord);1438 std::unique_ptr<ValueProfData> VPData =1439 ValueProfData::serializeFrom(SrcRecord);1440 1441 NamedInstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2});1442 InstrProfSymtab Symtab;1443 Symtab.mapAddress(uint64_t(callee1), 0x1000ULL);1444 Symtab.mapAddress(uint64_t(callee2), 0x2000ULL);1445 Symtab.mapAddress(uint64_t(callee3), 0x3000ULL);1446 Symtab.mapAddress(uint64_t(callee4), 0x4000ULL);1447 // Missing mapping for callee51448 1449 auto getVTableStartAddr = [](const uint64_t *vtable) -> uint64_t {1450 return uint64_t(vtable);1451 };1452 auto getVTableEndAddr = [](const uint64_t *vtable) -> uint64_t {1453 return uint64_t(vtable) + 16;1454 };1455 auto getVTableMidAddr = [](const uint64_t *vtable) -> uint64_t {1456 return uint64_t(vtable) + 8;1457 };1458 // vtable1, vtable2, vtable3, vtable4 get mapped; vtable5, vtable6 are not1459 // mapped.1460 Symtab.mapVTableAddress(getVTableStartAddr(vtable1),1461 getVTableEndAddr(vtable1), MD5Hash("vtable1"));1462 Symtab.mapVTableAddress(getVTableStartAddr(vtable2),1463 getVTableEndAddr(vtable2), MD5Hash("vtable2"));1464 Symtab.mapVTableAddress(getVTableStartAddr(vtable3),1465 getVTableEndAddr(vtable3), MD5Hash("vtable3"));1466 Symtab.mapVTableAddress(getVTableStartAddr(vtable4),1467 getVTableEndAddr(vtable4), MD5Hash("vtable4"));1468 1469 VPData->deserializeTo(Record, &Symtab);1470 1471 // Now read data from Record and sanity check the data1472 ASSERT_EQ(Record.getNumValueSites(IPVK_IndirectCallTarget), 6U);1473 1474 // Look up the value correpsonding to the middle of a vtable in symtab and1475 // test that it's the hash of the name.1476 EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable1)),1477 MD5Hash("vtable1"));1478 EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable2)),1479 MD5Hash("vtable2"));1480 EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable3)),1481 MD5Hash("vtable3"));1482 EXPECT_EQ(Symtab.getVTableHashFromAddress(getVTableMidAddr(vtable4)),1483 MD5Hash("vtable4"));1484 1485 auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {1486 return VD1.Count > VD2.Count;1487 };1488 SmallVector<InstrProfValueData> VD_0(1489 Record.getValueArrayForSite(IPVK_IndirectCallTarget, 0));1490 ASSERT_THAT(VD_0, SizeIs(5));1491 llvm::sort(VD_0, Cmp);1492 ASSERT_EQ(VD_0[0].Value, 0x2000ULL);1493 ASSERT_EQ(VD_0[0].Count, 1000U);1494 ASSERT_EQ(VD_0[1].Value, 0x3000ULL);1495 ASSERT_EQ(VD_0[1].Count, 500U);1496 ASSERT_EQ(VD_0[2].Value, 0x1000ULL);1497 ASSERT_EQ(VD_0[2].Count, 400U);1498 1499 // callee5 does not have a mapped value -- default to 0.1500 ASSERT_EQ(VD_0[4].Value, 0ULL);1501 1502 // Sanity check the vtable value data1503 ASSERT_EQ(Record.getNumValueSites(IPVK_VTableTarget), 4U);1504 1505 {1506 // The first vtable site.1507 SmallVector<InstrProfValueData> VD(1508 Record.getValueArrayForSite(IPVK_VTableTarget, 0));1509 ASSERT_THAT(VD, SizeIs(5));1510 llvm::sort(VD, Cmp);1511 EXPECT_EQ(VD[0].Count, 1000U);1512 EXPECT_EQ(VD[0].Value, MD5Hash("vtable2"));1513 EXPECT_EQ(VD[1].Count, 500U);1514 EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));1515 EXPECT_EQ(VD[2].Value, MD5Hash("vtable1"));1516 EXPECT_EQ(VD[2].Count, 400U);1517 EXPECT_EQ(VD[3].Value, MD5Hash("vtable4"));1518 EXPECT_EQ(VD[3].Count, 300U);1519 1520 // vtable5 isn't mapped -- default to 0.1521 EXPECT_EQ(VD[4].Value, 0U);1522 EXPECT_EQ(VD[4].Count, 100U);1523 }1524 1525 {1526 // The second vtable site.1527 SmallVector<InstrProfValueData> VD(1528 Record.getValueArrayForSite(IPVK_VTableTarget, 1));1529 ASSERT_THAT(VD, SizeIs(4));1530 llvm::sort(VD, Cmp);1531 EXPECT_EQ(VD[0].Value, MD5Hash("vtable2"));1532 EXPECT_EQ(VD[0].Count, 2500U);1533 EXPECT_EQ(VD[1].Value, MD5Hash("vtable1"));1534 EXPECT_EQ(VD[1].Count, 1300U);1535 1536 EXPECT_EQ(VD[2].Value, MD5Hash("vtable3"));1537 EXPECT_EQ(VD[2].Count, 1000U);1538 // vtable5 isn't mapped -- default to 0.1539 EXPECT_EQ(VD[3].Value, 0U);1540 EXPECT_EQ(VD[3].Count, 800U);1541 }1542 1543 {1544 // The third vtable site.1545 SmallVector<InstrProfValueData> VD(1546 Record.getValueArrayForSite(IPVK_VTableTarget, 2));1547 ASSERT_THAT(VD, SizeIs(3));1548 llvm::sort(VD, Cmp);1549 EXPECT_EQ(VD[0].Count, 5500U);1550 EXPECT_EQ(VD[0].Value, MD5Hash("vtable4"));1551 EXPECT_EQ(VD[1].Count, 1000U);1552 EXPECT_EQ(VD[1].Value, MD5Hash("vtable3"));1553 // vtable6 isn't mapped -- default to 0.1554 EXPECT_EQ(VD[2].Value, 0U);1555 EXPECT_EQ(VD[2].Count, 800U);1556 }1557 1558 {1559 // The fourth vtable site.1560 SmallVector<InstrProfValueData> VD(1561 Record.getValueArrayForSite(IPVK_VTableTarget, 3));1562 ASSERT_THAT(VD, SizeIs(2));1563 llvm::sort(VD, Cmp);1564 EXPECT_EQ(VD[0].Count, 2000U);1565 EXPECT_EQ(VD[0].Value, MD5Hash("vtable3"));1566 EXPECT_EQ(VD[1].Count, 1800U);1567 EXPECT_EQ(VD[1].Value, MD5Hash("vtable2"));1568 }1569}1570 1571TEST_P(MaybeSparseInstrProfTest, get_max_function_count) {1572 Writer.addRecord({"foo", 0x1234, {1ULL << 31, 2}}, Err);1573 Writer.addRecord({"bar", 0, {1ULL << 63}}, Err);1574 Writer.addRecord({"baz", 0x5678, {0, 0, 0, 0}}, Err);1575 auto Profile = Writer.writeBuffer();1576 readProfile(std::move(Profile));1577 1578 ASSERT_EQ(1ULL << 63, Reader->getMaximumFunctionCount(/* IsCS */ false));1579}1580 1581TEST_P(MaybeSparseInstrProfTest, get_weighted_function_counts) {1582 Writer.addRecord({"foo", 0x1234, {1, 2}}, 3, Err);1583 Writer.addRecord({"foo", 0x1235, {3, 4}}, 5, Err);1584 auto Profile = Writer.writeBuffer();1585 readProfile(std::move(Profile));1586 1587 std::vector<uint64_t> Counts;1588 EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1234, Counts),1589 Succeeded());1590 ASSERT_EQ(2U, Counts.size());1591 ASSERT_EQ(3U, Counts[0]);1592 ASSERT_EQ(6U, Counts[1]);1593 1594 EXPECT_THAT_ERROR(Reader->getFunctionCounts("foo", 0x1235, Counts),1595 Succeeded());1596 ASSERT_EQ(2U, Counts.size());1597 ASSERT_EQ(15U, Counts[0]);1598 ASSERT_EQ(20U, Counts[1]);1599}1600 1601// Testing symtab creator interface used by indexed profile reader.1602TEST(SymtabTest, instr_prof_symtab_test) {1603 std::vector<StringRef> FuncNames;1604 FuncNames.push_back("func1");1605 FuncNames.push_back("func2");1606 FuncNames.push_back("func3");1607 FuncNames.push_back("bar1");1608 FuncNames.push_back("bar2");1609 FuncNames.push_back("bar3");1610 InstrProfSymtab Symtab;1611 EXPECT_THAT_ERROR(Symtab.create(FuncNames), Succeeded());1612 StringRef R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func1"));1613 ASSERT_EQ(StringRef("func1"), R);1614 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func2"));1615 ASSERT_EQ(StringRef("func2"), R);1616 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func3"));1617 ASSERT_EQ(StringRef("func3"), R);1618 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar1"));1619 ASSERT_EQ(StringRef("bar1"), R);1620 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar2"));1621 ASSERT_EQ(StringRef("bar2"), R);1622 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar3"));1623 ASSERT_EQ(StringRef("bar3"), R);1624 1625 // negative tests1626 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar4"));1627 ASSERT_EQ(StringRef(), R);1628 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("foo4"));1629 ASSERT_EQ(StringRef(), R);1630 1631 // Now incrementally update the symtab1632 EXPECT_THAT_ERROR(Symtab.addFuncName("blah_1"), Succeeded());1633 EXPECT_THAT_ERROR(Symtab.addFuncName("blah_2"), Succeeded());1634 EXPECT_THAT_ERROR(Symtab.addFuncName("blah_3"), Succeeded());1635 1636 // Check again1637 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("blah_1"));1638 ASSERT_EQ(StringRef("blah_1"), R);1639 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("blah_2"));1640 ASSERT_EQ(StringRef("blah_2"), R);1641 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("blah_3"));1642 ASSERT_EQ(StringRef("blah_3"), R);1643 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func1"));1644 ASSERT_EQ(StringRef("func1"), R);1645 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func2"));1646 ASSERT_EQ(StringRef("func2"), R);1647 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("func3"));1648 ASSERT_EQ(StringRef("func3"), R);1649 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar1"));1650 ASSERT_EQ(StringRef("bar1"), R);1651 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar2"));1652 ASSERT_EQ(StringRef("bar2"), R);1653 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash("bar3"));1654 ASSERT_EQ(StringRef("bar3"), R);1655}1656 1657// Test that we get an error when creating a bogus symtab.1658TEST(SymtabTest, instr_prof_bogus_symtab_empty_func_name) {1659 InstrProfSymtab Symtab;1660 EXPECT_TRUE(ErrorEquals(instrprof_error::malformed, Symtab.addFuncName("")));1661}1662 1663// Testing symtab creator interface used by value profile transformer.1664TEST(SymtabTest, instr_prof_symtab_module_test) {1665 LLVMContext Ctx;1666 std::unique_ptr<Module> M = std::make_unique<Module>("MyModule.cpp", Ctx);1667 FunctionType *FTy = FunctionType::get(Type::getVoidTy(Ctx),1668 /*isVarArg=*/false);1669 Function::Create(FTy, Function::ExternalLinkage, "Gfoo", M.get());1670 Function::Create(FTy, Function::ExternalLinkage, "Gblah", M.get());1671 Function::Create(FTy, Function::ExternalLinkage, "Gbar", M.get());1672 Function::Create(FTy, Function::InternalLinkage, "Ifoo", M.get());1673 Function::Create(FTy, Function::InternalLinkage, "Iblah", M.get());1674 Function::Create(FTy, Function::InternalLinkage, "Ibar", M.get());1675 Function::Create(FTy, Function::PrivateLinkage, "Pfoo", M.get());1676 Function::Create(FTy, Function::PrivateLinkage, "Pblah", M.get());1677 Function::Create(FTy, Function::PrivateLinkage, "Pbar", M.get());1678 Function::Create(FTy, Function::WeakODRLinkage, "Wfoo", M.get());1679 Function::Create(FTy, Function::WeakODRLinkage, "Wblah", M.get());1680 Function::Create(FTy, Function::WeakODRLinkage, "Wbar", M.get());1681 1682 // [ptr, ptr, ptr]1683 ArrayType *VTableArrayType = ArrayType::get(1684 PointerType::get(Ctx, M->getDataLayout().getDefaultGlobalsAddressSpace()),1685 3);1686 Constant *Int32TyNull =1687 llvm::ConstantExpr::getNullValue(PointerType::getUnqual(Ctx));1688 SmallVector<llvm::Type *, 1> tys = {VTableArrayType};1689 StructType *VTableType = llvm::StructType::get(Ctx, tys);1690 1691 // Create two vtables in the module, one with external linkage and the other1692 // with local linkage.1693 for (auto [Name, Linkage] :1694 {std::pair{"ExternalGV", GlobalValue::ExternalLinkage},1695 {"LocalGV", GlobalValue::InternalLinkage}}) {1696 llvm::Twine FuncName(Name, StringRef("VFunc"));1697 Function *VFunc = Function::Create(FTy, Linkage, FuncName, M.get());1698 GlobalVariable *GV = new llvm::GlobalVariable(1699 *M, VTableType, /* isConstant= */ true, Linkage,1700 llvm::ConstantStruct::get(1701 VTableType,1702 {llvm::ConstantArray::get(VTableArrayType,1703 {Int32TyNull, Int32TyNull, VFunc})}),1704 Name);1705 // Add type metadata for the test data, since vtables with type metadata1706 // are added to symtab.1707 GV->addTypeMetadata(16, MDString::get(Ctx, Name));1708 }1709 1710 InstrProfSymtab ProfSymtab;1711 EXPECT_THAT_ERROR(ProfSymtab.create(*M), Succeeded());1712 1713 StringRef Funcs[] = {"Gfoo", "Gblah", "Gbar", "Ifoo", "Iblah", "Ibar",1714 "Pfoo", "Pblah", "Pbar", "Wfoo", "Wblah", "Wbar"};1715 1716 for (unsigned I = 0; I < std::size(Funcs); I++) {1717 Function *F = M->getFunction(Funcs[I]);1718 1719 std::string IRPGOName = getIRPGOFuncName(*F);1720 auto IRPGOFuncName =1721 ProfSymtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(IRPGOName));1722 EXPECT_EQ(IRPGOName, IRPGOFuncName);1723 EXPECT_EQ(Funcs[I], getParsedIRPGOName(IRPGOFuncName).second);1724 // Ensure we can still read this old record name.1725 std::string PGOName = getPGOFuncName(*F);1726 auto PGOFuncName =1727 ProfSymtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(PGOName));1728 EXPECT_EQ(PGOName, PGOFuncName);1729 EXPECT_THAT(PGOFuncName.str(), EndsWith(Funcs[I].str()));1730 }1731 1732 for (auto [VTableName, PGOName] : {std::pair{"ExternalGV", "ExternalGV"},1733 {"LocalGV", "MyModule.cpp;LocalGV"}}) {1734 GlobalVariable *GV =1735 M->getGlobalVariable(VTableName, /* AllowInternal=*/true);1736 1737 // Test that ProfSymtab returns the expected name given a hash.1738 std::string IRPGOName = getPGOName(*GV);1739 EXPECT_STREQ(IRPGOName.c_str(), PGOName);1740 uint64_t GUID = IndexedInstrProf::ComputeHash(IRPGOName);1741 EXPECT_EQ(IRPGOName, ProfSymtab.getFuncOrVarName(GUID));1742 EXPECT_EQ(VTableName, getParsedIRPGOName(IRPGOName).second);1743 1744 // Test that ProfSymtab returns the expected global variable1745 EXPECT_EQ(GV, ProfSymtab.getGlobalVariable(GUID));1746 }1747}1748 1749// Testing symtab serialization and creator/deserialization interface1750// used by coverage map reader, and raw profile reader.1751TEST(SymtabTest, instr_prof_symtab_compression_test) {1752 std::vector<std::string> FuncNames1;1753 std::vector<std::string> FuncNames2;1754 for (int I = 0; I < 3; I++) {1755 std::string str;1756 raw_string_ostream OS(str);1757 OS << "func_" << I;1758 FuncNames1.push_back(OS.str());1759 str.clear();1760 OS << "f oooooooooooooo_" << I;1761 FuncNames1.push_back(OS.str());1762 str.clear();1763 OS << "BAR_" << I;1764 FuncNames2.push_back(OS.str());1765 str.clear();1766 OS << "BlahblahBlahblahBar_" << I;1767 FuncNames2.push_back(OS.str());1768 }1769 1770 for (bool DoCompression : {false, true}) {1771 // Compressing:1772 std::string FuncNameStrings1;1773 EXPECT_THAT_ERROR(collectGlobalObjectNameStrings(1774 FuncNames1,1775 (DoCompression && compression::zlib::isAvailable()),1776 FuncNameStrings1),1777 Succeeded());1778 1779 // Compressing:1780 std::string FuncNameStrings2;1781 EXPECT_THAT_ERROR(collectGlobalObjectNameStrings(1782 FuncNames2,1783 (DoCompression && compression::zlib::isAvailable()),1784 FuncNameStrings2),1785 Succeeded());1786 1787 for (int Padding = 0; Padding < 2; Padding++) {1788 // Join with paddings :1789 std::string FuncNameStrings = FuncNameStrings1;1790 for (int P = 0; P < Padding; P++) {1791 FuncNameStrings.push_back('\0');1792 }1793 FuncNameStrings += FuncNameStrings2;1794 1795 // Now decompress:1796 InstrProfSymtab Symtab;1797 EXPECT_THAT_ERROR(Symtab.create(StringRef(FuncNameStrings)), Succeeded());1798 1799 // Now do the checks:1800 // First sampling some data points:1801 StringRef R =1802 Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(FuncNames1[0]));1803 ASSERT_EQ(StringRef("func_0"), R);1804 R = Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(FuncNames1[1]));1805 ASSERT_EQ(StringRef("f oooooooooooooo_0"), R);1806 for (int I = 0; I < 3; I++) {1807 std::string N[4];1808 N[0] = FuncNames1[2 * I];1809 N[1] = FuncNames1[2 * I + 1];1810 N[2] = FuncNames2[2 * I];1811 N[3] = FuncNames2[2 * I + 1];1812 for (int J = 0; J < 4; J++) {1813 StringRef R =1814 Symtab.getFuncOrVarName(IndexedInstrProf::ComputeHash(N[J]));1815 ASSERT_EQ(StringRef(N[J]), R);1816 }1817 }1818 }1819 }1820}1821 1822TEST_P(MaybeSparseInstrProfTest, remapping_test) {1823 Writer.addRecord({"_Z3fooi", 0x1234, {1, 2, 3, 4}}, Err);1824 Writer.addRecord({"file;_Z3barf", 0x567, {5, 6, 7}}, Err);1825 auto Profile = Writer.writeBuffer();1826 readProfile(std::move(Profile), llvm::MemoryBuffer::getMemBuffer(R"(1827 type i l1828 name 3bar 4quux1829 )"));1830 1831 std::vector<uint64_t> Counts;1832 for (StringRef FooName : {"_Z3fooi", "_Z3fool"}) {1833 EXPECT_THAT_ERROR(Reader->getFunctionCounts(FooName, 0x1234, Counts),1834 Succeeded());1835 ASSERT_EQ(4u, Counts.size());1836 EXPECT_EQ(1u, Counts[0]);1837 EXPECT_EQ(2u, Counts[1]);1838 EXPECT_EQ(3u, Counts[2]);1839 EXPECT_EQ(4u, Counts[3]);1840 }1841 1842 for (StringRef BarName : {"file;_Z3barf", "file;_Z4quuxf"}) {1843 EXPECT_THAT_ERROR(Reader->getFunctionCounts(BarName, 0x567, Counts),1844 Succeeded());1845 ASSERT_EQ(3u, Counts.size());1846 EXPECT_EQ(5u, Counts[0]);1847 EXPECT_EQ(6u, Counts[1]);1848 EXPECT_EQ(7u, Counts[2]);1849 }1850 1851 for (StringRef BadName : {"_Z3foof", "_Z4quuxi", "_Z3barl", "", "_ZZZ",1852 "_Z3barf", "otherfile:_Z4quuxf"}) {1853 EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x1234, Counts),1854 Failed());1855 EXPECT_THAT_ERROR(Reader->getFunctionCounts(BadName, 0x567, Counts),1856 Failed());1857 }1858}1859 1860TEST_F(SparseInstrProfTest, preserve_no_records) {1861 Writer.addRecord({"foo", 0x1234, {0}}, Err);1862 Writer.addRecord({"bar", 0x4321, {0, 0}}, Err);1863 Writer.addRecord({"baz", 0x4321, {0, 0, 0}}, Err);1864 1865 auto Profile = Writer.writeBuffer();1866 readProfile(std::move(Profile));1867 1868 auto I = Reader->begin(), E = Reader->end();1869 ASSERT_TRUE(I == E);1870}1871 1872INSTANTIATE_TEST_SUITE_P(MaybeSparse, MaybeSparseInstrProfTest,1873 ::testing::Bool());1874 1875#if defined(_LP64) && defined(EXPENSIVE_CHECKS)1876TEST(ProfileReaderTest, ReadsLargeFiles) {1877 const size_t LargeSize = 1ULL << 32; // 4GB1878 1879 auto RawProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);1880 if (!RawProfile)1881 GTEST_SKIP();1882 auto RawProfileReaderOrErr = InstrProfReader::create(std::move(RawProfile));1883 ASSERT_TRUE(1884 std::get<0>(InstrProfError::take(RawProfileReaderOrErr.takeError())) ==1885 instrprof_error::unrecognized_format);1886 1887 auto IndexedProfile = WritableMemoryBuffer::getNewUninitMemBuffer(LargeSize);1888 if (!IndexedProfile)1889 GTEST_SKIP();1890 auto IndexedReaderOrErr =1891 IndexedInstrProfReader::create(std::move(IndexedProfile), nullptr);1892 ASSERT_TRUE(1893 std::get<0>(InstrProfError::take(IndexedReaderOrErr.takeError())) ==1894 instrprof_error::bad_magic);1895}1896#endif1897 1898} // end anonymous namespace1899