818 lines · cpp
1//===- MemoryProfileInfoTest.cpp - Memory Profile Info Unit Tests-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "llvm/Analysis/MemoryProfileInfo.h"10#include "llvm/AsmParser/Parser.h"11#include "llvm/IR/Constants.h"12#include "llvm/IR/Instructions.h"13#include "llvm/IR/LLVMContext.h"14#include "llvm/IR/Module.h"15#include "llvm/IR/ModuleSummaryIndex.h"16#include "llvm/Support/CommandLine.h"17#include "llvm/Support/Compiler.h"18#include "llvm/Support/SourceMgr.h"19#include "gmock/gmock.h"20#include "gtest/gtest.h"21#include <cstring>22#include <sys/types.h>23 24using namespace llvm;25using namespace llvm::memprof;26 27namespace llvm {28LLVM_ABI extern cl::opt<bool> MemProfKeepAllNotColdContexts;29} // end namespace llvm30 31namespace {32 33class MemoryProfileInfoTest : public testing::Test {34protected:35 std::unique_ptr<Module> makeLLVMModule(LLVMContext &C, const char *IR) {36 SMDiagnostic Err;37 std::unique_ptr<Module> Mod = parseAssemblyString(IR, Err, C);38 if (!Mod)39 Err.print("MemoryProfileInfoTest", errs());40 return Mod;41 }42 43 std::unique_ptr<ModuleSummaryIndex> makeLLVMIndex(const char *Summary) {44 SMDiagnostic Err;45 std::unique_ptr<ModuleSummaryIndex> Index =46 parseSummaryIndexAssemblyString(Summary, Err);47 if (!Index)48 Err.print("MemoryProfileInfoTest", errs());49 return Index;50 }51 52 // This looks for a call that has the given value name, which53 // is the name of the value being assigned the call return value.54 CallBase *findCall(Function &F, const char *Name = nullptr) {55 for (auto &BB : F)56 for (auto &I : BB)57 if (auto *CB = dyn_cast<CallBase>(&I))58 if (!Name || CB->getName() == Name)59 return CB;60 return nullptr;61 }62};63 64// Test the hasSingleAllocType helper.65TEST_F(MemoryProfileInfoTest, SingleAllocType) {66 uint8_t NotCold = (uint8_t)AllocationType::NotCold;67 uint8_t Cold = (uint8_t)AllocationType::Cold;68 uint8_t Hot = (uint8_t)AllocationType::Hot;69 EXPECT_TRUE(hasSingleAllocType(NotCold));70 EXPECT_TRUE(hasSingleAllocType(Cold));71 EXPECT_TRUE(hasSingleAllocType(Hot));72 EXPECT_FALSE(hasSingleAllocType(NotCold | Cold));73 EXPECT_FALSE(hasSingleAllocType(NotCold | Hot));74 EXPECT_FALSE(hasSingleAllocType(Cold | Hot));75 EXPECT_FALSE(hasSingleAllocType(NotCold | Cold | Hot));76}77 78// Test buildCallstackMetadata helper.79TEST_F(MemoryProfileInfoTest, BuildCallStackMD) {80 LLVMContext C;81 MDNode *CallStack = buildCallstackMetadata({1, 2, 3}, C);82 ASSERT_EQ(CallStack->getNumOperands(), 3u);83 unsigned ExpectedId = 1;84 for (auto &Op : CallStack->operands()) {85 auto *StackId = mdconst::dyn_extract<ConstantInt>(Op);86 EXPECT_EQ(StackId->getZExtValue(), ExpectedId++);87 }88}89 90// Test CallStackTrie::addCallStack interface taking allocation type and list of91// call stack ids.92// Check that allocations with a single allocation type along all call stacks93// get an attribute instead of memprof metadata.94TEST_F(MemoryProfileInfoTest, Attribute) {95 LLVMContext C;96 std::unique_ptr<Module> M = makeLLVMModule(C,97 R"IR(98target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"99target triple = "x86_64-pc-linux-gnu"100define i32* @test() {101entry:102 %call1 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)103 %0 = bitcast i8* %call1 to i32*104 %call2 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)105 %1 = bitcast i8* %call2 to i32*106 %call3 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)107 %2 = bitcast i8* %call3 to i32* 108 %call4 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)109 %3 = bitcast i8* %call4 to i32*110 ret i32* %1111}112declare dso_local noalias noundef i8* @malloc(i64 noundef)113)IR");114 115 Function *Func = M->getFunction("test");116 117 // First call has all cold contexts.118 CallStackTrie Trie1;119 Trie1.addCallStack(AllocationType::Cold, {1, 2});120 Trie1.addCallStack(AllocationType::Cold, {1, 3, 4});121 CallBase *Call1 = findCall(*Func, "call1");122 Trie1.buildAndAttachMIBMetadata(Call1);123 124 EXPECT_FALSE(Call1->hasMetadata(LLVMContext::MD_memprof));125 EXPECT_TRUE(Call1->hasFnAttr("memprof"));126 EXPECT_EQ(Call1->getFnAttr("memprof").getValueAsString(), "cold");127 128 // Second call has all non-cold contexts.129 CallStackTrie Trie2;130 Trie2.addCallStack(AllocationType::NotCold, {5, 6});131 Trie2.addCallStack(AllocationType::NotCold, {5, 7, 8});132 CallBase *Call2 = findCall(*Func, "call2");133 Trie2.buildAndAttachMIBMetadata(Call2);134 135 EXPECT_FALSE(Call2->hasMetadata(LLVMContext::MD_memprof));136 EXPECT_TRUE(Call2->hasFnAttr("memprof"));137 EXPECT_EQ(Call2->getFnAttr("memprof").getValueAsString(), "notcold");138 139 // Third call has all hot contexts.140 CallStackTrie Trie3;141 Trie3.addCallStack(AllocationType::Hot, {9, 10});142 Trie3.addCallStack(AllocationType::Hot, {9, 11, 12});143 CallBase *Call3 = findCall(*Func, "call3");144 Trie3.buildAndAttachMIBMetadata(Call3);145 146 EXPECT_FALSE(Call3->hasMetadata(LLVMContext::MD_memprof));147 EXPECT_TRUE(Call3->hasFnAttr("memprof"));148 EXPECT_EQ(Call3->getFnAttr("memprof").getValueAsString(), "hot");149 150 // Fourth call has hot and non-cold contexts. These should be treated as151 // notcold and given a notcold attribute.152 CallStackTrie Trie4;153 Trie4.addCallStack(AllocationType::Hot, {5, 6});154 Trie4.addCallStack(AllocationType::NotCold, {5, 7, 8});155 CallBase *Call4 = findCall(*Func, "call4");156 Trie4.buildAndAttachMIBMetadata(Call4);157 158 EXPECT_FALSE(Call4->hasMetadata(LLVMContext::MD_memprof));159 EXPECT_TRUE(Call4->hasFnAttr("memprof"));160 EXPECT_EQ(Call4->getFnAttr("memprof").getValueAsString(), "notcold");161}162 163// TODO: Use this matcher in existing tests.164// ExpectedVals should be a vector of expected MIBs and their allocation type165// and stack id contents in order, of type:166// std::vector<std::pair<AllocationType, std::vector<unsigned>>>167MATCHER_P(MemprofMetadataEquals, ExpectedVals, "Matching !memprof contents") {168 auto PrintAndFail = [&]() {169 std::string Buffer;170 llvm::raw_string_ostream OS(Buffer);171 OS << "Expected:\n";172 for (auto &[ExpectedAllocType, ExpectedStackIds] : ExpectedVals) {173 OS << "\t" << getAllocTypeAttributeString(ExpectedAllocType) << " { ";174 for (auto Id : ExpectedStackIds)175 OS << Id << " ";176 OS << "}\n";177 }178 OS << "Got:\n";179 arg->printTree(OS);180 *result_listener << "!memprof metadata differs!\n" << Buffer;181 return false;182 };183 184 if (ExpectedVals.size() != arg->getNumOperands())185 return PrintAndFail();186 187 for (size_t I = 0; I < ExpectedVals.size(); I++) {188 const auto &[ExpectedAllocType, ExpectedStackIds] = ExpectedVals[I];189 MDNode *MIB = dyn_cast<MDNode>(arg->getOperand(I));190 if (getMIBAllocType(MIB) != ExpectedAllocType)191 return PrintAndFail();192 MDNode *StackMD = getMIBStackNode(MIB);193 EXPECT_NE(StackMD, nullptr);194 if (StackMD->getNumOperands() != ExpectedStackIds.size())195 return PrintAndFail();196 for (size_t J = 0; J < ExpectedStackIds.size(); J++) {197 auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(J));198 if (StackId->getZExtValue() != ExpectedStackIds[J])199 return PrintAndFail();200 }201 }202 return true;203}204 205// Test CallStackTrie::addCallStack interface taking allocation type and list of206// call stack ids.207// Test that an allocation call reached by both cold and non cold call stacks208// gets memprof metadata representing the different allocation type contexts.209TEST_F(MemoryProfileInfoTest, ColdAndNotColdMIB) {210 LLVMContext C;211 std::unique_ptr<Module> M = makeLLVMModule(C,212 R"IR(213target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"214target triple = "x86_64-pc-linux-gnu"215define i32* @test() {216entry:217 %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)218 %0 = bitcast i8* %call to i32*219 ret i32* %0220}221declare dso_local noalias noundef i8* @malloc(i64 noundef)222)IR");223 224 Function *Func = M->getFunction("test");225 226 CallStackTrie Trie;227 Trie.addCallStack(AllocationType::Cold, {1, 2});228 Trie.addCallStack(AllocationType::NotCold, {1, 3});229 230 CallBase *Call = findCall(*Func, "call");231 Trie.buildAndAttachMIBMetadata(Call);232 233 EXPECT_TRUE(Call->hasFnAttr("memprof"));234 EXPECT_EQ(Call->getFnAttr("memprof").getValueAsString(), "ambiguous");235 EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));236 MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);237 ASSERT_EQ(MemProfMD->getNumOperands(), 2u);238 for (auto &MIBOp : MemProfMD->operands()) {239 MDNode *MIB = dyn_cast<MDNode>(MIBOp);240 MDNode *StackMD = getMIBStackNode(MIB);241 ASSERT_NE(StackMD, nullptr);242 ASSERT_EQ(StackMD->getNumOperands(), 2u);243 auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));244 ASSERT_EQ(StackId->getZExtValue(), 1u);245 StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));246 if (StackId->getZExtValue() == 2u)247 EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);248 else {249 ASSERT_EQ(StackId->getZExtValue(), 3u);250 EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);251 }252 }253}254 255// Test CallStackTrie::addCallStack interface taking allocation type and list of256// call stack ids.257// Test that an allocation call reached by both cold and hot call stacks258// gets memprof metadata representing the different allocation type contexts.259TEST_F(MemoryProfileInfoTest, ColdAndHotMIB) {260 LLVMContext C;261 std::unique_ptr<Module> M = makeLLVMModule(C,262 R"IR(263target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"264target triple = "x86_64-pc-linux-gnu"265define i32* @test() {266entry:267 %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)268 %0 = bitcast i8* %call to i32*269 ret i32* %0270}271declare dso_local noalias noundef i8* @malloc(i64 noundef)272)IR");273 274 Function *Func = M->getFunction("test");275 276 CallStackTrie Trie;277 Trie.addCallStack(AllocationType::Cold, {1, 2});278 Trie.addCallStack(AllocationType::Hot, {1, 3});279 280 CallBase *Call = findCall(*Func, "call");281 Trie.buildAndAttachMIBMetadata(Call);282 283 EXPECT_TRUE(Call->hasFnAttr("memprof"));284 EXPECT_EQ(Call->getFnAttr("memprof").getValueAsString(), "ambiguous");285 EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));286 MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);287 ASSERT_EQ(MemProfMD->getNumOperands(), 2u);288 for (auto &MIBOp : MemProfMD->operands()) {289 MDNode *MIB = dyn_cast<MDNode>(MIBOp);290 MDNode *StackMD = getMIBStackNode(MIB);291 ASSERT_NE(StackMD, nullptr);292 ASSERT_EQ(StackMD->getNumOperands(), 2u);293 auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));294 ASSERT_EQ(StackId->getZExtValue(), 1u);295 StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));296 if (StackId->getZExtValue() == 2u)297 EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);298 else {299 ASSERT_EQ(StackId->getZExtValue(), 3u);300 // Hot contexts are converted to NotCold when building the metadata.301 EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);302 }303 }304}305 306// Test CallStackTrie::addCallStack interface taking allocation type and list of307// call stack ids.308// Test that an allocation call reached by both cold, non cold and hot call309// stacks gets memprof metadata representing the different allocation type310// contexts.311TEST_F(MemoryProfileInfoTest, ColdAndNotColdAndHotMIB) {312 LLVMContext C;313 std::unique_ptr<Module> M = makeLLVMModule(C,314 R"IR(315target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"316target triple = "x86_64-pc-linux-gnu"317define i32* @test() {318entry:319 %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)320 %0 = bitcast i8* %call to i32*321 ret i32* %0322}323declare dso_local noalias noundef i8* @malloc(i64 noundef)324)IR");325 326 Function *Func = M->getFunction("test");327 328 CallStackTrie Trie;329 Trie.addCallStack(AllocationType::Cold, {1, 2});330 Trie.addCallStack(AllocationType::NotCold, {1, 3});331 // This will be pruned as it is unnecessary to determine how to clone the332 // cold allocation.333 Trie.addCallStack(AllocationType::Hot, {1, 4});334 335 CallBase *Call = findCall(*Func, "call");336 Trie.buildAndAttachMIBMetadata(Call);337 338 EXPECT_TRUE(Call->hasFnAttr("memprof"));339 EXPECT_EQ(Call->getFnAttr("memprof").getValueAsString(), "ambiguous");340 EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));341 MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);342 ASSERT_EQ(MemProfMD->getNumOperands(), 2u);343 for (auto &MIBOp : MemProfMD->operands()) {344 MDNode *MIB = dyn_cast<MDNode>(MIBOp);345 MDNode *StackMD = getMIBStackNode(MIB);346 ASSERT_NE(StackMD, nullptr);347 ASSERT_EQ(StackMD->getNumOperands(), 2u);348 auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));349 ASSERT_EQ(StackId->getZExtValue(), 1u);350 StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));351 if (StackId->getZExtValue() == 2u) {352 EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);353 } else if (StackId->getZExtValue() == 3u) {354 EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);355 }356 }357}358 359// Test CallStackTrie::addCallStack interface taking allocation type and list of360// call stack ids.361// Test that an allocation call reached by multiple call stacks has memprof362// metadata with the contexts trimmed to the minimum context required to363// identify the allocation type.364TEST_F(MemoryProfileInfoTest, TrimmedMIBContext) {365 LLVMContext C;366 std::unique_ptr<Module> M = makeLLVMModule(C,367 R"IR(368target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"369target triple = "x86_64-pc-linux-gnu"370define i32* @test() {371entry:372 %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)373 %0 = bitcast i8* %call to i32*374 ret i32* %0375}376declare dso_local noalias noundef i8* @malloc(i64 noundef)377)IR");378 379 Function *Func = M->getFunction("test");380 381 CallStackTrie Trie;382 // We should be able to trim the following two and combine into a single MIB383 // with the cold context {1, 2}.384 Trie.addCallStack(AllocationType::Cold, {1, 2, 3});385 Trie.addCallStack(AllocationType::Cold, {1, 2, 4});386 // We should be able to trim the following two and combine into a single MIB387 // with the non-cold context {1, 5}.388 Trie.addCallStack(AllocationType::NotCold, {1, 5, 6});389 Trie.addCallStack(AllocationType::NotCold, {1, 5, 7});390 // These will be pruned as they are unnecessary to determine how to clone the391 // cold allocation.392 Trie.addCallStack(AllocationType::Hot, {1, 8, 9});393 Trie.addCallStack(AllocationType::Hot, {1, 8, 10});394 395 CallBase *Call = findCall(*Func, "call");396 Trie.buildAndAttachMIBMetadata(Call);397 398 EXPECT_TRUE(Call->hasFnAttr("memprof"));399 EXPECT_EQ(Call->getFnAttr("memprof").getValueAsString(), "ambiguous");400 EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));401 MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);402 ASSERT_EQ(MemProfMD->getNumOperands(), 2u);403 for (auto &MIBOp : MemProfMD->operands()) {404 MDNode *MIB = dyn_cast<MDNode>(MIBOp);405 MDNode *StackMD = getMIBStackNode(MIB);406 ASSERT_NE(StackMD, nullptr);407 ASSERT_EQ(StackMD->getNumOperands(), 2u);408 auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));409 EXPECT_EQ(StackId->getZExtValue(), 1u);410 StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));411 if (StackId->getZExtValue() == 2u)412 EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);413 else if (StackId->getZExtValue() == 5u)414 EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);415 }416}417 418// Test to ensure that we prune NotCold contexts that are unneeded for419// determining where Cold contexts need to be cloned to enable correct hinting.420TEST_F(MemoryProfileInfoTest, PruneUnneededNotColdContexts) {421 LLVMContext C;422 std::unique_ptr<Module> M = makeLLVMModule(C,423 R"IR(424target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"425target triple = "x86_64-pc-linux-gnu"426define i32* @test() {427entry:428 %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)429 %0 = bitcast i8* %call to i32*430 ret i32* %0431}432declare dso_local noalias noundef i8* @malloc(i64 noundef)433)IR");434 435 Function *Func = M->getFunction("test");436 437 CallStackTrie Trie;438 439 Trie.addCallStack(AllocationType::Cold, {1, 2, 3, 4});440 Trie.addCallStack(AllocationType::Cold, {1, 2, 3, 5, 6, 7});441 // This NotCold context is needed to know where the above two Cold contexts442 // must be cloned from:443 Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 5, 6, 13});444 445 Trie.addCallStack(AllocationType::Cold, {1, 2, 3, 8, 9, 10});446 // This NotCold context is needed to know where the above Cold context must be447 // cloned from:448 Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 8, 9, 14});449 // This NotCold context is not needed since the above is sufficient (we pick450 // the first in sorted order).451 Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 8, 9, 15});452 453 // None of these NotCold contexts are needed as the Cold contexts they454 // overlap with are covered by longer overlapping NotCold contexts.455 Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 12});456 Trie.addCallStack(AllocationType::NotCold, {1, 2, 11});457 Trie.addCallStack(AllocationType::NotCold, {1, 16});458 459 std::vector<std::pair<AllocationType, std::vector<unsigned>>> ExpectedVals = {460 {AllocationType::Cold, {1, 2, 3, 4}},461 {AllocationType::Cold, {1, 2, 3, 5, 6, 7}},462 {AllocationType::NotCold, {1, 2, 3, 5, 6, 13}},463 {AllocationType::Cold, {1, 2, 3, 8, 9, 10}},464 {AllocationType::NotCold, {1, 2, 3, 8, 9, 14}}};465 466 CallBase *Call = findCall(*Func, "call");467 ASSERT_NE(Call, nullptr);468 Trie.buildAndAttachMIBMetadata(Call);469 470 EXPECT_TRUE(Call->hasFnAttr("memprof"));471 EXPECT_EQ(Call->getFnAttr("memprof").getValueAsString(), "ambiguous");472 EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));473 MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);474 EXPECT_THAT(MemProfMD, MemprofMetadataEquals(ExpectedVals));475}476 477// Test to ensure that we keep optionally keep unneeded NotCold contexts.478// Same as PruneUnneededNotColdContexts test but with the479// MemProfKeepAllNotColdContexts set to true.480TEST_F(MemoryProfileInfoTest, KeepUnneededNotColdContexts) {481 LLVMContext C;482 std::unique_ptr<Module> M = makeLLVMModule(C,483 R"IR(484target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"485target triple = "x86_64-pc-linux-gnu"486define i32* @test() {487entry:488 %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40)489 %0 = bitcast i8* %call to i32*490 ret i32* %0491}492declare dso_local noalias noundef i8* @malloc(i64 noundef)493)IR");494 495 Function *Func = M->getFunction("test");496 497 CallStackTrie Trie;498 499 Trie.addCallStack(AllocationType::Cold, {1, 2, 3, 4});500 Trie.addCallStack(AllocationType::Cold, {1, 2, 3, 5, 6, 7});501 // This NotCold context is needed to know where the above two Cold contexts502 // must be cloned from:503 Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 5, 6, 13});504 505 Trie.addCallStack(AllocationType::Cold, {1, 2, 3, 8, 9, 10});506 // This NotCold context is needed to know where the above Cold context must be507 // cloned from:508 Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 8, 9, 14});509 // This NotCold context is not needed since the above is sufficient (we pick510 // the first in sorted order).511 Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 8, 9, 15});512 513 // None of these NotCold contexts are needed as the Cold contexts they514 // overlap with are covered by longer overlapping NotCold contexts.515 Trie.addCallStack(AllocationType::NotCold, {1, 2, 3, 12});516 Trie.addCallStack(AllocationType::NotCold, {1, 2, 11});517 Trie.addCallStack(AllocationType::NotCold, {1, 16});518 519 // We should keep all of the above contexts, even those that are unneeded by520 // default, because we will set the option to keep them.521 std::vector<std::pair<AllocationType, std::vector<unsigned>>> ExpectedVals = {522 {AllocationType::Cold, {1, 2, 3, 4}},523 {AllocationType::Cold, {1, 2, 3, 5, 6, 7}},524 {AllocationType::NotCold, {1, 2, 3, 5, 6, 13}},525 {AllocationType::Cold, {1, 2, 3, 8, 9, 10}},526 {AllocationType::NotCold, {1, 2, 3, 8, 9, 14}},527 {AllocationType::NotCold, {1, 2, 3, 8, 9, 15}},528 {AllocationType::NotCold, {1, 2, 3, 12}},529 {AllocationType::NotCold, {1, 2, 11}},530 {AllocationType::NotCold, {1, 16}}};531 532 CallBase *Call = findCall(*Func, "call");533 ASSERT_NE(Call, nullptr);534 535 // Specify that all non-cold contexts should be kept.536 bool OrigMemProfKeepAllNotColdContexts = MemProfKeepAllNotColdContexts;537 MemProfKeepAllNotColdContexts = true;538 539 Trie.buildAndAttachMIBMetadata(Call);540 541 // Restore original option value.542 MemProfKeepAllNotColdContexts = OrigMemProfKeepAllNotColdContexts;543 544 EXPECT_TRUE(Call->hasFnAttr("memprof"));545 EXPECT_EQ(Call->getFnAttr("memprof").getValueAsString(), "ambiguous");546 EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));547 MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);548 EXPECT_THAT(MemProfMD, MemprofMetadataEquals(ExpectedVals));549}550 551// Test CallStackTrie::addCallStack interface taking memprof MIB metadata.552// Check that allocations annotated with memprof metadata with a single553// allocation type get simplified to an attribute.554TEST_F(MemoryProfileInfoTest, SimplifyMIBToAttribute) {555 LLVMContext C;556 std::unique_ptr<Module> M = makeLLVMModule(C,557 R"IR(558target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"559target triple = "x86_64-pc-linux-gnu"560define i32* @test() {561entry:562 %call1 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !0563 %0 = bitcast i8* %call1 to i32*564 %call2 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !3565 %1 = bitcast i8* %call2 to i32*566 %call3 = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !6567 %2 = bitcast i8* %call3 to i32*568 ret i32* %1569}570declare dso_local noalias noundef i8* @malloc(i64 noundef)571!0 = !{!1}572!1 = !{!2, !"cold"}573!2 = !{i64 1, i64 2, i64 3}574!3 = !{!4}575!4 = !{!5, !"notcold"}576!5 = !{i64 4, i64 5, i64 6, i64 7}577!6 = !{!7}578!7 = !{!8, !"hot"}579!8 = !{i64 8, i64 9, i64 10}580)IR");581 582 Function *Func = M->getFunction("test");583 584 // First call has all cold contexts.585 CallStackTrie Trie1;586 CallBase *Call1 = findCall(*Func, "call1");587 MDNode *MemProfMD1 = Call1->getMetadata(LLVMContext::MD_memprof);588 ASSERT_EQ(MemProfMD1->getNumOperands(), 1u);589 MDNode *MIB1 = dyn_cast<MDNode>(MemProfMD1->getOperand(0));590 Trie1.addCallStack(MIB1);591 Trie1.buildAndAttachMIBMetadata(Call1);592 593 EXPECT_TRUE(Call1->hasFnAttr("memprof"));594 EXPECT_EQ(Call1->getFnAttr("memprof").getValueAsString(), "cold");595 596 // Second call has all non-cold contexts.597 CallStackTrie Trie2;598 CallBase *Call2 = findCall(*Func, "call2");599 MDNode *MemProfMD2 = Call2->getMetadata(LLVMContext::MD_memprof);600 ASSERT_EQ(MemProfMD2->getNumOperands(), 1u);601 MDNode *MIB2 = dyn_cast<MDNode>(MemProfMD2->getOperand(0));602 Trie2.addCallStack(MIB2);603 Trie2.buildAndAttachMIBMetadata(Call2);604 605 EXPECT_TRUE(Call2->hasFnAttr("memprof"));606 EXPECT_EQ(Call2->getFnAttr("memprof").getValueAsString(), "notcold");607 608 // Third call has all hot contexts.609 CallStackTrie Trie3;610 CallBase *Call3 = findCall(*Func, "call3");611 MDNode *MemProfMD3 = Call3->getMetadata(LLVMContext::MD_memprof);612 ASSERT_EQ(MemProfMD2->getNumOperands(), 1u);613 MDNode *MIB3 = dyn_cast<MDNode>(MemProfMD3->getOperand(0));614 Trie3.addCallStack(MIB3);615 Trie3.buildAndAttachMIBMetadata(Call3);616 617 EXPECT_TRUE(Call3->hasFnAttr("memprof"));618 EXPECT_EQ(Call3->getFnAttr("memprof").getValueAsString(), "hot");619}620 621// Test CallStackTrie::addCallStack interface taking memprof MIB metadata.622// Test that allocations annotated with memprof metadata with multiple call623// stacks gets new memprof metadata with the contexts trimmed to the minimum624// context required to identify the allocation type.625TEST_F(MemoryProfileInfoTest, ReTrimMIBContext) {626 LLVMContext C;627 std::unique_ptr<Module> M = makeLLVMModule(C,628 R"IR(629target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"630target triple = "x86_64-pc-linux-gnu"631define i32* @test() {632entry:633 %call = call noalias dereferenceable_or_null(40) i8* @malloc(i64 noundef 40), !memprof !0634 %0 = bitcast i8* %call to i32*635 ret i32* %0636}637declare dso_local noalias noundef i8* @malloc(i64 noundef)638!0 = !{!1, !3, !5, !7, !9, !11}639!1 = !{!2, !"cold"}640!2 = !{i64 1, i64 2, i64 3}641!3 = !{!4, !"cold", !13}642!4 = !{i64 1, i64 2, i64 4}643!5 = !{!6, !"notcold"}644!6 = !{i64 1, i64 5, i64 6}645!7 = !{!8, !"notcold"}646!8 = !{i64 1, i64 5, i64 7}647!9 = !{!10, !"hot"}648!10 = !{i64 1, i64 8, i64 9}649!11 = !{!12, !"hot"}650!12 = !{i64 1, i64 8, i64 10} 651!13 = !{i64 123, i64 456}652)IR");653 654 Function *Func = M->getFunction("test");655 656 CallStackTrie Trie;657 ASSERT_TRUE(Trie.empty());658 CallBase *Call = findCall(*Func, "call");659 MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);660 for (auto &MIBOp : MemProfMD->operands()) {661 MDNode *MIB = dyn_cast<MDNode>(MIBOp);662 Trie.addCallStack(MIB);663 }664 ASSERT_FALSE(Trie.empty());665 Trie.buildAndAttachMIBMetadata(Call);666 667 // We should be able to trim the first two and combine into a single MIB668 // with the cold context {1, 2}.669 // We should be able to trim the second two and combine into a single MIB670 // with the non-cold context {1, 5}.671 // The hot allocations will be converted to NotCold and pruned as they672 // are unnecessary to determine how to clone the cold allocation.673 674 EXPECT_TRUE(Call->hasFnAttr("memprof"));675 EXPECT_EQ(Call->getFnAttr("memprof").getValueAsString(), "ambiguous");676 EXPECT_TRUE(Call->hasMetadata(LLVMContext::MD_memprof));677 MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);678 ASSERT_EQ(MemProfMD->getNumOperands(), 2u);679 for (auto &MIBOp : MemProfMD->operands()) {680 MDNode *MIB = dyn_cast<MDNode>(MIBOp);681 MDNode *StackMD = getMIBStackNode(MIB);682 ASSERT_NE(StackMD, nullptr);683 ASSERT_EQ(StackMD->getNumOperands(), 2u);684 auto *StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(0));685 EXPECT_EQ(StackId->getZExtValue(), 1u);686 StackId = mdconst::dyn_extract<ConstantInt>(StackMD->getOperand(1));687 if (StackId->getZExtValue() == 2u) {688 EXPECT_EQ(getMIBAllocType(MIB), AllocationType::Cold);689 // We should propagate the single context size info from the second cold690 // context above onto the new merged/trimmed context.691 ASSERT_EQ(MIB->getNumOperands(), 3u);692 MDNode *ContextSizePair = dyn_cast<MDNode>(MIB->getOperand(2));693 assert(ContextSizePair->getNumOperands() == 2);694 EXPECT_EQ(695 mdconst::dyn_extract<ConstantInt>(ContextSizePair->getOperand(0))696 ->getZExtValue(),697 123u);698 EXPECT_EQ(699 mdconst::dyn_extract<ConstantInt>(ContextSizePair->getOperand(1))700 ->getZExtValue(),701 456u);702 } else if (StackId->getZExtValue() == 5u) {703 EXPECT_EQ(getMIBAllocType(MIB), AllocationType::NotCold);704 ASSERT_EQ(MIB->getNumOperands(), 2u);705 }706 }707}708 709TEST_F(MemoryProfileInfoTest, CallStackTestIR) {710 LLVMContext C;711 std::unique_ptr<Module> M = makeLLVMModule(C,712 R"IR(713target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"714target triple = "x86_64-pc-linux-gnu"715define ptr @test() {716entry:717 %call = call noalias noundef nonnull dereferenceable(10) ptr @_Znam(i64 noundef 10), !memprof !1, !callsite !6718 ret ptr %call719}720declare noundef nonnull ptr @_Znam(i64 noundef)721!1 = !{!2, !4, !7}722!2 = !{!3, !"notcold"}723!3 = !{i64 1, i64 2, i64 3, i64 4}724!4 = !{!5, !"cold"}725!5 = !{i64 1, i64 2, i64 3, i64 5}726!6 = !{i64 1}727!7 = !{!8, !"hot"}728!8 = !{i64 1, i64 2, i64 3, i64 6} 729)IR");730 731 Function *Func = M->getFunction("test");732 CallBase *Call = findCall(*Func, "call");733 734 CallStack<MDNode, MDNode::op_iterator> InstCallsite(735 Call->getMetadata(LLVMContext::MD_callsite));736 737 MDNode *MemProfMD = Call->getMetadata(LLVMContext::MD_memprof);738 unsigned Idx = 0;739 for (auto &MIBOp : MemProfMD->operands()) {740 auto *MIBMD = cast<const MDNode>(MIBOp);741 MDNode *StackNode = getMIBStackNode(MIBMD);742 CallStack<MDNode, MDNode::op_iterator> StackContext(StackNode);743 EXPECT_EQ(StackContext.back(), 4 + Idx);744 std::vector<uint64_t> StackIds;745 for (auto ContextIter = StackContext.beginAfterSharedPrefix(InstCallsite);746 ContextIter != StackContext.end(); ++ContextIter)747 StackIds.push_back(*ContextIter);748 if (Idx == 0) {749 std::vector<uint64_t> Expected = {2, 3, 4};750 EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));751 } else if (Idx == 1) {752 std::vector<uint64_t> Expected = {2, 3, 5};753 EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));754 } else {755 std::vector<uint64_t> Expected = {2, 3, 6};756 EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));757 }758 Idx++;759 }760}761 762TEST_F(MemoryProfileInfoTest, CallStackTestSummary) {763 std::unique_ptr<ModuleSummaryIndex> Index = makeLLVMIndex(R"Summary(764^0 = module: (path: "test.o", hash: (0, 0, 0, 0, 0))765^1 = gv: (guid: 23, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 2, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 0, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), allocs: ((versions: (none), memProf: ((type: notcold, stackIds: (1, 2, 3, 4)), (type: cold, stackIds: (1, 2, 3, 5)), (type: hot, stackIds: (1, 2, 3, 6))))))))766^2 = gv: (guid: 25, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 0, dsoLocal: 1, canAutoHide: 0), insts: 22, funcFlags: (readNone: 0, readOnly: 0, noRecurse: 1, returnDoesNotAlias: 0, noInline: 1, alwaysInline: 0, noUnwind: 0, mayThrow: 0, hasUnknownCall: 0, mustBeUnreachable: 0), calls: ((callee: ^1)), callsites: ((callee: ^1, clones: (0), stackIds: (3, 4)), (callee: ^1, clones: (0), stackIds: (3, 5)), (callee: ^1, clones: (0), stackIds: (3, 6))))))767)Summary");768 769 ASSERT_NE(Index, nullptr);770 auto *CallsiteSummary =771 cast<FunctionSummary>(Index->getGlobalValueSummary(/*guid=*/25));772 unsigned Idx = 0;773 for (auto &CI : CallsiteSummary->callsites()) {774 CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator> InstCallsite(775 &CI);776 std::vector<uint64_t> StackIds;777 for (auto StackIdIndex : InstCallsite)778 StackIds.push_back(Index->getStackIdAtIndex(StackIdIndex));779 if (Idx == 0) {780 std::vector<uint64_t> Expected = {3, 4};781 EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));782 } else if (Idx == 1) {783 std::vector<uint64_t> Expected = {3, 5};784 EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));785 } else {786 std::vector<uint64_t> Expected = {3, 6};787 EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));788 }789 Idx++;790 }791 792 auto *AllocSummary =793 cast<FunctionSummary>(Index->getGlobalValueSummary(/*guid=*/23));794 for (auto &AI : AllocSummary->allocs()) {795 unsigned Idx = 0;796 for (auto &MIB : AI.MIBs) {797 CallStack<MIBInfo, SmallVector<unsigned>::const_iterator> StackContext(798 &MIB);799 EXPECT_EQ(Index->getStackIdAtIndex(StackContext.back()), 4 + Idx);800 std::vector<uint64_t> StackIds;801 for (auto StackIdIndex : StackContext)802 StackIds.push_back(Index->getStackIdAtIndex(StackIdIndex));803 if (Idx == 0) {804 std::vector<uint64_t> Expected = {1, 2, 3, 4};805 EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));806 } else if (Idx == 1) {807 std::vector<uint64_t> Expected = {1, 2, 3, 5};808 EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));809 } else {810 std::vector<uint64_t> Expected = {1, 2, 3, 6};811 EXPECT_EQ(ArrayRef(StackIds), ArrayRef(Expected));812 }813 Idx++;814 }815 }816}817} // end anonymous namespace818