893 lines · cpp
1//===- MemProfUse.cpp - memory allocation profile use pass --*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the MemProfUsePass which reads memory profiling data10// and uses it to add metadata to instructions to guide optimization.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/Transforms/Instrumentation/MemProfUse.h"15#include "llvm/ADT/SmallVector.h"16#include "llvm/ADT/Statistic.h"17#include "llvm/ADT/StringRef.h"18#include "llvm/Analysis/MemoryProfileInfo.h"19#include "llvm/Analysis/OptimizationRemarkEmitter.h"20#include "llvm/Analysis/StaticDataProfileInfo.h"21#include "llvm/Analysis/TargetLibraryInfo.h"22#include "llvm/IR/DiagnosticInfo.h"23#include "llvm/IR/Function.h"24#include "llvm/IR/IntrinsicInst.h"25#include "llvm/IR/Module.h"26#include "llvm/ProfileData/DataAccessProf.h"27#include "llvm/ProfileData/InstrProf.h"28#include "llvm/ProfileData/InstrProfReader.h"29#include "llvm/ProfileData/MemProfCommon.h"30#include "llvm/Support/BLAKE3.h"31#include "llvm/Support/CommandLine.h"32#include "llvm/Support/Debug.h"33#include "llvm/Support/HashBuilder.h"34#include "llvm/Support/VirtualFileSystem.h"35#include "llvm/Transforms/Utils/LongestCommonSequence.h"36#include <map>37#include <set>38 39using namespace llvm;40using namespace llvm::memprof;41 42#define DEBUG_TYPE "memprof"43 44namespace llvm {45extern cl::opt<bool> PGOWarnMissing;46extern cl::opt<bool> NoPGOWarnMismatch;47extern cl::opt<bool> NoPGOWarnMismatchComdatWeak;48} // namespace llvm49 50// By default disable matching of allocation profiles onto operator new that51// already explicitly pass a hot/cold hint, since we don't currently52// override these hints anyway.53static cl::opt<bool> ClMemProfMatchHotColdNew(54 "memprof-match-hot-cold-new",55 cl::desc(56 "Match allocation profiles onto existing hot/cold operator new calls"),57 cl::Hidden, cl::init(false));58 59static cl::opt<bool>60 ClPrintMemProfMatchInfo("memprof-print-match-info",61 cl::desc("Print matching stats for each allocation "62 "context in this module's profiles"),63 cl::Hidden, cl::init(false));64 65static cl::opt<bool>66 SalvageStaleProfile("memprof-salvage-stale-profile",67 cl::desc("Salvage stale MemProf profile"),68 cl::init(false), cl::Hidden);69 70static cl::opt<bool> ClMemProfAttachCalleeGuids(71 "memprof-attach-calleeguids",72 cl::desc(73 "Attach calleeguids as value profile metadata for indirect calls."),74 cl::init(true), cl::Hidden);75 76static cl::opt<unsigned> MinMatchedColdBytePercent(77 "memprof-matching-cold-threshold", cl::init(100), cl::Hidden,78 cl::desc("Min percent of cold bytes matched to hint allocation cold"));79 80static cl::opt<bool> AnnotateStaticDataSectionPrefix(81 "memprof-annotate-static-data-prefix", cl::init(false), cl::Hidden,82 cl::desc("If true, annotate the static data section prefix"));83 84// Matching statistics85STATISTIC(NumOfMemProfMissing, "Number of functions without memory profile.");86STATISTIC(NumOfMemProfMismatch,87 "Number of functions having mismatched memory profile hash.");88STATISTIC(NumOfMemProfFunc, "Number of functions having valid memory profile.");89STATISTIC(NumOfMemProfAllocContextProfiles,90 "Number of alloc contexts in memory profile.");91STATISTIC(NumOfMemProfCallSiteProfiles,92 "Number of callsites in memory profile.");93STATISTIC(NumOfMemProfMatchedAllocContexts,94 "Number of matched memory profile alloc contexts.");95STATISTIC(NumOfMemProfMatchedAllocs,96 "Number of matched memory profile allocs.");97STATISTIC(NumOfMemProfMatchedCallSites,98 "Number of matched memory profile callsites.");99STATISTIC(NumOfMemProfHotGlobalVars,100 "Number of global vars annotated with 'hot' section prefix.");101STATISTIC(NumOfMemProfColdGlobalVars,102 "Number of global vars annotated with 'unlikely' section prefix.");103STATISTIC(NumOfMemProfUnknownGlobalVars,104 "Number of global vars with unknown hotness (no section prefix).");105STATISTIC(NumOfMemProfExplicitSectionGlobalVars,106 "Number of global vars with user-specified section (not annotated).");107 108static void addCallsiteMetadata(Instruction &I,109 ArrayRef<uint64_t> InlinedCallStack,110 LLVMContext &Ctx) {111 I.setMetadata(LLVMContext::MD_callsite,112 buildCallstackMetadata(InlinedCallStack, Ctx));113}114 115static uint64_t computeStackId(GlobalValue::GUID Function, uint32_t LineOffset,116 uint32_t Column) {117 llvm::HashBuilder<llvm::TruncatedBLAKE3<8>, llvm::endianness::little>118 HashBuilder;119 HashBuilder.add(Function, LineOffset, Column);120 llvm::BLAKE3Result<8> Hash = HashBuilder.final();121 uint64_t Id;122 std::memcpy(&Id, Hash.data(), sizeof(Hash));123 return Id;124}125 126static uint64_t computeStackId(const memprof::Frame &Frame) {127 return computeStackId(Frame.Function, Frame.LineOffset, Frame.Column);128}129 130static AllocationType getAllocType(const AllocationInfo *AllocInfo) {131 return getAllocType(AllocInfo->Info.getTotalLifetimeAccessDensity(),132 AllocInfo->Info.getAllocCount(),133 AllocInfo->Info.getTotalLifetime());134}135 136static AllocationType addCallStack(CallStackTrie &AllocTrie,137 const AllocationInfo *AllocInfo,138 uint64_t FullStackId) {139 SmallVector<uint64_t> StackIds;140 for (const auto &StackFrame : AllocInfo->CallStack)141 StackIds.push_back(computeStackId(StackFrame));142 auto AllocType = getAllocType(AllocInfo);143 std::vector<ContextTotalSize> ContextSizeInfo;144 if (recordContextSizeInfoForAnalysis()) {145 auto TotalSize = AllocInfo->Info.getTotalSize();146 assert(TotalSize);147 assert(FullStackId != 0);148 ContextSizeInfo.push_back({FullStackId, TotalSize});149 }150 AllocTrie.addCallStack(AllocType, StackIds, std::move(ContextSizeInfo));151 return AllocType;152}153 154// Return true if InlinedCallStack, computed from a call instruction's debug155// info, is a prefix of ProfileCallStack, a list of Frames from profile data156// (either the allocation data or a callsite).157static bool158stackFrameIncludesInlinedCallStack(ArrayRef<Frame> ProfileCallStack,159 ArrayRef<uint64_t> InlinedCallStack) {160 return ProfileCallStack.size() >= InlinedCallStack.size() &&161 llvm::equal(ProfileCallStack.take_front(InlinedCallStack.size()),162 InlinedCallStack, [](const Frame &F, uint64_t StackId) {163 return computeStackId(F) == StackId;164 });165}166 167static bool isAllocationWithHotColdVariant(const Function *Callee,168 const TargetLibraryInfo &TLI) {169 if (!Callee)170 return false;171 LibFunc Func;172 if (!TLI.getLibFunc(*Callee, Func))173 return false;174 switch (Func) {175 case LibFunc_Znwm:176 case LibFunc_ZnwmRKSt9nothrow_t:177 case LibFunc_ZnwmSt11align_val_t:178 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:179 case LibFunc_Znam:180 case LibFunc_ZnamRKSt9nothrow_t:181 case LibFunc_ZnamSt11align_val_t:182 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:183 case LibFunc_size_returning_new:184 case LibFunc_size_returning_new_aligned:185 return true;186 case LibFunc_Znwm12__hot_cold_t:187 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:188 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:189 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:190 case LibFunc_Znam12__hot_cold_t:191 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:192 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:193 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:194 case LibFunc_size_returning_new_hot_cold:195 case LibFunc_size_returning_new_aligned_hot_cold:196 return ClMemProfMatchHotColdNew;197 default:198 return false;199 }200}201 202static void HandleUnsupportedAnnotationKinds(GlobalVariable &GVar,203 AnnotationKind Kind) {204 assert(Kind != llvm::memprof::AnnotationKind::AnnotationOK &&205 "Should not handle AnnotationOK here");206 SmallString<32> Reason;207 switch (Kind) {208 case llvm::memprof::AnnotationKind::ExplicitSection:209 ++NumOfMemProfExplicitSectionGlobalVars;210 Reason.append("explicit section name");211 break;212 case llvm::memprof::AnnotationKind::DeclForLinker:213 Reason.append("linker declaration");214 break;215 case llvm::memprof::AnnotationKind::ReservedName:216 Reason.append("name starts with `llvm.`");217 break;218 default:219 llvm_unreachable("Unexpected annotation kind");220 }221 LLVM_DEBUG(dbgs() << "Skip annotation for " << GVar.getName() << " due to "222 << Reason << ".\n");223}224 225struct AllocMatchInfo {226 uint64_t TotalSize = 0;227 AllocationType AllocType = AllocationType::None;228};229 230DenseMap<uint64_t, SmallVector<CallEdgeTy, 0>>231memprof::extractCallsFromIR(Module &M, const TargetLibraryInfo &TLI,232 function_ref<bool(uint64_t)> IsPresentInProfile) {233 DenseMap<uint64_t, SmallVector<CallEdgeTy, 0>> Calls;234 235 auto GetOffset = [](const DILocation *DIL) {236 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &237 0xffff;238 };239 240 for (Function &F : M) {241 if (F.isDeclaration())242 continue;243 244 for (auto &BB : F) {245 for (auto &I : BB) {246 if (!isa<CallBase>(&I) || isa<IntrinsicInst>(&I))247 continue;248 249 auto *CB = dyn_cast<CallBase>(&I);250 auto *CalledFunction = CB->getCalledFunction();251 // Disregard indirect calls and intrinsics.252 if (!CalledFunction || CalledFunction->isIntrinsic())253 continue;254 255 StringRef CalleeName = CalledFunction->getName();256 // True if we are calling a heap allocation function that supports257 // hot/cold variants.258 bool IsAlloc = isAllocationWithHotColdVariant(CalledFunction, TLI);259 // True for the first iteration below, indicating that we are looking at260 // a leaf node.261 bool IsLeaf = true;262 for (const DILocation *DIL = I.getDebugLoc(); DIL;263 DIL = DIL->getInlinedAt()) {264 StringRef CallerName = DIL->getSubprogramLinkageName();265 assert(!CallerName.empty() &&266 "Be sure to enable -fdebug-info-for-profiling");267 uint64_t CallerGUID = memprof::getGUID(CallerName);268 uint64_t CalleeGUID = memprof::getGUID(CalleeName);269 // Pretend that we are calling a function with GUID == 0 if we are270 // in the inline stack leading to a heap allocation function.271 if (IsAlloc) {272 if (IsLeaf) {273 // For leaf nodes, set CalleeGUID to 0 without consulting274 // IsPresentInProfile.275 CalleeGUID = 0;276 } else if (!IsPresentInProfile(CalleeGUID)) {277 // In addition to the leaf case above, continue to set CalleeGUID278 // to 0 as long as we don't see CalleeGUID in the profile.279 CalleeGUID = 0;280 } else {281 // Once we encounter a callee that exists in the profile, stop282 // setting CalleeGUID to 0.283 IsAlloc = false;284 }285 }286 287 LineLocation Loc = {GetOffset(DIL), DIL->getColumn()};288 Calls[CallerGUID].emplace_back(Loc, CalleeGUID);289 CalleeName = CallerName;290 IsLeaf = false;291 }292 }293 }294 }295 296 // Sort each call list by the source location.297 for (auto &[CallerGUID, CallList] : Calls) {298 llvm::sort(CallList);299 CallList.erase(llvm::unique(CallList), CallList.end());300 }301 302 return Calls;303}304 305DenseMap<uint64_t, LocToLocMap>306memprof::computeUndriftMap(Module &M, IndexedInstrProfReader *MemProfReader,307 const TargetLibraryInfo &TLI) {308 DenseMap<uint64_t, LocToLocMap> UndriftMaps;309 310 DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>> CallsFromProfile =311 MemProfReader->getMemProfCallerCalleePairs();312 DenseMap<uint64_t, SmallVector<memprof::CallEdgeTy, 0>> CallsFromIR =313 extractCallsFromIR(M, TLI, [&](uint64_t GUID) {314 return CallsFromProfile.contains(GUID);315 });316 317 // Compute an undrift map for each CallerGUID.318 for (const auto &[CallerGUID, IRAnchors] : CallsFromIR) {319 auto It = CallsFromProfile.find(CallerGUID);320 if (It == CallsFromProfile.end())321 continue;322 const auto &ProfileAnchors = It->second;323 324 LocToLocMap Matchings;325 longestCommonSequence<LineLocation, GlobalValue::GUID>(326 ProfileAnchors, IRAnchors, std::equal_to<GlobalValue::GUID>(),327 [&](LineLocation A, LineLocation B) { Matchings.try_emplace(A, B); });328 [[maybe_unused]] bool Inserted =329 UndriftMaps.try_emplace(CallerGUID, std::move(Matchings)).second;330 331 // The insertion must succeed because we visit each GUID exactly once.332 assert(Inserted);333 }334 335 return UndriftMaps;336}337 338// Given a MemProfRecord, undrift all the source locations present in the339// record in place.340static void341undriftMemProfRecord(const DenseMap<uint64_t, LocToLocMap> &UndriftMaps,342 memprof::MemProfRecord &MemProfRec) {343 // Undrift a call stack in place.344 auto UndriftCallStack = [&](std::vector<Frame> &CallStack) {345 for (auto &F : CallStack) {346 auto I = UndriftMaps.find(F.Function);347 if (I == UndriftMaps.end())348 continue;349 auto J = I->second.find(LineLocation(F.LineOffset, F.Column));350 if (J == I->second.end())351 continue;352 auto &NewLoc = J->second;353 F.LineOffset = NewLoc.LineOffset;354 F.Column = NewLoc.Column;355 }356 };357 358 for (auto &AS : MemProfRec.AllocSites)359 UndriftCallStack(AS.CallStack);360 361 for (auto &CS : MemProfRec.CallSites)362 UndriftCallStack(CS.Frames);363}364 365// Helper function to process CalleeGuids and create value profile metadata366static void addVPMetadata(Module &M, Instruction &I,367 ArrayRef<GlobalValue::GUID> CalleeGuids) {368 if (!ClMemProfAttachCalleeGuids || CalleeGuids.empty())369 return;370 371 if (I.getMetadata(LLVMContext::MD_prof)) {372 uint64_t Unused;373 // TODO: When merging is implemented, increase this to a typical ICP value374 // (e.g., 3-6) For now, we only need to check if existing data exists, so 1375 // is sufficient376 auto ExistingVD = getValueProfDataFromInst(I, IPVK_IndirectCallTarget,377 /*MaxNumValueData=*/1, Unused);378 // We don't know how to merge value profile data yet.379 if (!ExistingVD.empty()) {380 return;381 }382 }383 384 SmallVector<InstrProfValueData, 4> VDs;385 uint64_t TotalCount = 0;386 387 for (const GlobalValue::GUID CalleeGUID : CalleeGuids) {388 InstrProfValueData VD;389 VD.Value = CalleeGUID;390 // For MemProf, we don't have actual call counts, so we assign391 // a weight of 1 to each potential target.392 // TODO: Consider making this weight configurable or increasing it to393 // improve effectiveness for ICP.394 VD.Count = 1;395 VDs.push_back(VD);396 TotalCount += VD.Count;397 }398 399 if (!VDs.empty()) {400 annotateValueSite(M, I, VDs, TotalCount, IPVK_IndirectCallTarget,401 VDs.size());402 }403}404 405static void406handleAllocSite(Instruction &I, CallBase *CI,407 ArrayRef<uint64_t> InlinedCallStack, LLVMContext &Ctx,408 OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize,409 const std::set<const AllocationInfo *> &AllocInfoSet,410 std::map<std::pair<uint64_t, unsigned>, AllocMatchInfo>411 &FullStackIdToAllocMatchInfo) {412 // TODO: Remove this once the profile creation logic deduplicates contexts413 // that are the same other than the IsInlineFrame bool. Until then, keep the414 // largest.415 DenseMap<uint64_t, const AllocationInfo *> UniqueFullContextIdAllocInfo;416 for (auto *AllocInfo : AllocInfoSet) {417 auto FullStackId = computeFullStackId(AllocInfo->CallStack);418 auto [It, Inserted] =419 UniqueFullContextIdAllocInfo.insert({FullStackId, AllocInfo});420 // If inserted entry, done.421 if (Inserted)422 continue;423 // Keep the larger one, or the noncold one if they are the same size.424 auto CurSize = It->second->Info.getTotalSize();425 auto NewSize = AllocInfo->Info.getTotalSize();426 if ((CurSize > NewSize) ||427 (CurSize == NewSize &&428 getAllocType(AllocInfo) != AllocationType::NotCold))429 continue;430 It->second = AllocInfo;431 }432 // We may match this instruction's location list to multiple MIB433 // contexts. Add them to a Trie specialized for trimming the contexts to434 // the minimal needed to disambiguate contexts with unique behavior.435 CallStackTrie AllocTrie(&ORE, MaxColdSize);436 uint64_t TotalSize = 0;437 uint64_t TotalColdSize = 0;438 for (auto &[FullStackId, AllocInfo] : UniqueFullContextIdAllocInfo) {439 // Check the full inlined call stack against this one.440 // If we found and thus matched all frames on the call, include441 // this MIB.442 if (stackFrameIncludesInlinedCallStack(AllocInfo->CallStack,443 InlinedCallStack)) {444 NumOfMemProfMatchedAllocContexts++;445 auto AllocType = addCallStack(AllocTrie, AllocInfo, FullStackId);446 TotalSize += AllocInfo->Info.getTotalSize();447 if (AllocType == AllocationType::Cold)448 TotalColdSize += AllocInfo->Info.getTotalSize();449 // Record information about the allocation if match info printing450 // was requested.451 if (ClPrintMemProfMatchInfo) {452 assert(FullStackId != 0);453 FullStackIdToAllocMatchInfo[std::make_pair(FullStackId,454 InlinedCallStack.size())] = {455 AllocInfo->Info.getTotalSize(), AllocType};456 }457 }458 }459 // If the threshold for the percent of cold bytes is less than 100%,460 // and not all bytes are cold, see if we should still hint this461 // allocation as cold without context sensitivity.462 if (TotalColdSize < TotalSize && MinMatchedColdBytePercent < 100 &&463 TotalColdSize * 100 >= MinMatchedColdBytePercent * TotalSize) {464 AllocTrie.addSingleAllocTypeAttribute(CI, AllocationType::Cold, "dominant");465 return;466 }467 468 // We might not have matched any to the full inlined call stack.469 // But if we did, create and attach metadata, or a function attribute if470 // all contexts have identical profiled behavior.471 if (!AllocTrie.empty()) {472 NumOfMemProfMatchedAllocs++;473 // MemprofMDAttached will be false if a function attribute was474 // attached.475 bool MemprofMDAttached = AllocTrie.buildAndAttachMIBMetadata(CI);476 assert(MemprofMDAttached == I.hasMetadata(LLVMContext::MD_memprof));477 if (MemprofMDAttached) {478 // Add callsite metadata for the instruction's location list so that479 // it simpler later on to identify which part of the MIB contexts480 // are from this particular instruction (including during inlining,481 // when the callsite metadata will be updated appropriately).482 // FIXME: can this be changed to strip out the matching stack483 // context ids from the MIB contexts and not add any callsite484 // metadata here to save space?485 addCallsiteMetadata(I, InlinedCallStack, Ctx);486 }487 }488}489 490// Helper struct for maintaining refs to callsite data. As an alternative we491// could store a pointer to the CallSiteInfo struct but we also need the frame492// index. Using ArrayRefs instead makes it a little easier to read.493struct CallSiteEntry {494 // Subset of frames for the corresponding CallSiteInfo.495 ArrayRef<Frame> Frames;496 // Potential targets for indirect calls.497 ArrayRef<GlobalValue::GUID> CalleeGuids;498 499 // Only compare Frame contents.500 // Use pointer-based equality instead of ArrayRef's operator== which does501 // element-wise comparison. We want to check if it's the same slice of the502 // underlying array, not just equivalent content.503 bool operator==(const CallSiteEntry &Other) const {504 return Frames.data() == Other.Frames.data() &&505 Frames.size() == Other.Frames.size();506 }507};508 509struct CallSiteEntryHash {510 size_t operator()(const CallSiteEntry &Entry) const {511 return computeFullStackId(Entry.Frames);512 }513};514 515static void handleCallSite(516 Instruction &I, const Function *CalledFunction,517 ArrayRef<uint64_t> InlinedCallStack,518 const std::unordered_set<CallSiteEntry, CallSiteEntryHash> &CallSiteEntries,519 Module &M, std::set<std::vector<uint64_t>> &MatchedCallSites) {520 auto &Ctx = M.getContext();521 for (const auto &CallSiteEntry : CallSiteEntries) {522 // If we found and thus matched all frames on the call, create and523 // attach call stack metadata.524 if (stackFrameIncludesInlinedCallStack(CallSiteEntry.Frames,525 InlinedCallStack)) {526 NumOfMemProfMatchedCallSites++;527 addCallsiteMetadata(I, InlinedCallStack, Ctx);528 529 // Try to attach indirect call metadata if possible.530 if (!CalledFunction)531 addVPMetadata(M, I, CallSiteEntry.CalleeGuids);532 533 // Only need to find one with a matching call stack and add a single534 // callsite metadata.535 536 // Accumulate call site matching information upon request.537 if (ClPrintMemProfMatchInfo) {538 std::vector<uint64_t> CallStack;539 append_range(CallStack, InlinedCallStack);540 MatchedCallSites.insert(std::move(CallStack));541 }542 break;543 }544 }545}546 547static void readMemprof(Module &M, Function &F,548 IndexedInstrProfReader *MemProfReader,549 const TargetLibraryInfo &TLI,550 std::map<std::pair<uint64_t, unsigned>, AllocMatchInfo>551 &FullStackIdToAllocMatchInfo,552 std::set<std::vector<uint64_t>> &MatchedCallSites,553 DenseMap<uint64_t, LocToLocMap> &UndriftMaps,554 OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize) {555 auto &Ctx = M.getContext();556 // Previously we used getIRPGOFuncName() here. If F is local linkage,557 // getIRPGOFuncName() returns FuncName with prefix 'FileName;'. But558 // llvm-profdata uses FuncName in dwarf to create GUID which doesn't559 // contain FileName's prefix. It caused local linkage function can't560 // find MemProfRecord. So we use getName() now.561 // 'unique-internal-linkage-names' can make MemProf work better for local562 // linkage function.563 auto FuncName = F.getName();564 auto FuncGUID = Function::getGUIDAssumingExternalLinkage(FuncName);565 std::optional<memprof::MemProfRecord> MemProfRec;566 auto Err = MemProfReader->getMemProfRecord(FuncGUID).moveInto(MemProfRec);567 if (Err) {568 handleAllErrors(std::move(Err), [&](const InstrProfError &IPE) {569 auto Err = IPE.get();570 bool SkipWarning = false;571 LLVM_DEBUG(dbgs() << "Error in reading profile for Func " << FuncName572 << ": ");573 if (Err == instrprof_error::unknown_function) {574 NumOfMemProfMissing++;575 SkipWarning = !PGOWarnMissing;576 LLVM_DEBUG(dbgs() << "unknown function");577 } else if (Err == instrprof_error::hash_mismatch) {578 NumOfMemProfMismatch++;579 SkipWarning =580 NoPGOWarnMismatch ||581 (NoPGOWarnMismatchComdatWeak &&582 (F.hasComdat() ||583 F.getLinkage() == GlobalValue::AvailableExternallyLinkage));584 LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")");585 }586 587 if (SkipWarning)588 return;589 590 std::string Msg = (IPE.message() + Twine(" ") + F.getName().str() +591 Twine(" Hash = ") + std::to_string(FuncGUID))592 .str();593 594 Ctx.diagnose(595 DiagnosticInfoPGOProfile(M.getName().data(), Msg, DS_Warning));596 });597 return;598 }599 600 NumOfMemProfFunc++;601 602 // If requested, undrfit MemProfRecord so that the source locations in it603 // match those in the IR.604 if (SalvageStaleProfile)605 undriftMemProfRecord(UndriftMaps, *MemProfRec);606 607 // Detect if there are non-zero column numbers in the profile. If not,608 // treat all column numbers as 0 when matching (i.e. ignore any non-zero609 // columns in the IR). The profiled binary might have been built with610 // column numbers disabled, for example.611 bool ProfileHasColumns = false;612 613 // Build maps of the location hash to all profile data with that leaf location614 // (allocation info and the callsites).615 std::map<uint64_t, std::set<const AllocationInfo *>> LocHashToAllocInfo;616 617 // For the callsites we need to record slices of the frame array (see comments618 // below where the map entries are added) along with their CalleeGuids.619 std::map<uint64_t, std::unordered_set<CallSiteEntry, CallSiteEntryHash>>620 LocHashToCallSites;621 for (auto &AI : MemProfRec->AllocSites) {622 NumOfMemProfAllocContextProfiles++;623 // Associate the allocation info with the leaf frame. The later matching624 // code will match any inlined call sequences in the IR with a longer prefix625 // of call stack frames.626 uint64_t StackId = computeStackId(AI.CallStack[0]);627 LocHashToAllocInfo[StackId].insert(&AI);628 ProfileHasColumns |= AI.CallStack[0].Column;629 }630 for (auto &CS : MemProfRec->CallSites) {631 NumOfMemProfCallSiteProfiles++;632 // Need to record all frames from leaf up to and including this function,633 // as any of these may or may not have been inlined at this point.634 unsigned Idx = 0;635 for (auto &StackFrame : CS.Frames) {636 uint64_t StackId = computeStackId(StackFrame);637 ArrayRef<Frame> FrameSlice = ArrayRef<Frame>(CS.Frames).drop_front(Idx++);638 ArrayRef<GlobalValue::GUID> CalleeGuids(CS.CalleeGuids);639 LocHashToCallSites[StackId].insert({FrameSlice, CalleeGuids});640 641 ProfileHasColumns |= StackFrame.Column;642 // Once we find this function, we can stop recording.643 if (StackFrame.Function == FuncGUID)644 break;645 }646 assert(Idx <= CS.Frames.size() && CS.Frames[Idx - 1].Function == FuncGUID);647 }648 649 auto GetOffset = [](const DILocation *DIL) {650 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &651 0xffff;652 };653 654 // Now walk the instructions, looking up the associated profile data using655 // debug locations.656 for (auto &BB : F) {657 for (auto &I : BB) {658 if (I.isDebugOrPseudoInst())659 continue;660 // We are only interested in calls (allocation or interior call stack661 // context calls).662 auto *CI = dyn_cast<CallBase>(&I);663 if (!CI)664 continue;665 auto *CalledFunction = CI->getCalledFunction();666 if (CalledFunction && CalledFunction->isIntrinsic())667 continue;668 // List of call stack ids computed from the location hashes on debug669 // locations (leaf to inlined at root).670 SmallVector<uint64_t, 8> InlinedCallStack;671 // Was the leaf location found in one of the profile maps?672 bool LeafFound = false;673 // If leaf was found in a map, iterators pointing to its location in both674 // of the maps. It might exist in neither, one, or both (the latter case675 // can happen because we don't currently have discriminators to676 // distinguish the case when a single line/col maps to both an allocation677 // and another callsite).678 auto AllocInfoIter = LocHashToAllocInfo.end();679 auto CallSitesIter = LocHashToCallSites.end();680 for (const DILocation *DIL = I.getDebugLoc(); DIL != nullptr;681 DIL = DIL->getInlinedAt()) {682 // Use C++ linkage name if possible. Need to compile with683 // -fdebug-info-for-profiling to get linkage name.684 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();685 if (Name.empty())686 Name = DIL->getScope()->getSubprogram()->getName();687 auto CalleeGUID = Function::getGUIDAssumingExternalLinkage(Name);688 auto StackId = computeStackId(CalleeGUID, GetOffset(DIL),689 ProfileHasColumns ? DIL->getColumn() : 0);690 // Check if we have found the profile's leaf frame. If yes, collect691 // the rest of the call's inlined context starting here. If not, see if692 // we find a match further up the inlined context (in case the profile693 // was missing debug frames at the leaf).694 if (!LeafFound) {695 AllocInfoIter = LocHashToAllocInfo.find(StackId);696 CallSitesIter = LocHashToCallSites.find(StackId);697 if (AllocInfoIter != LocHashToAllocInfo.end() ||698 CallSitesIter != LocHashToCallSites.end())699 LeafFound = true;700 }701 if (LeafFound)702 InlinedCallStack.push_back(StackId);703 }704 // If leaf not in either of the maps, skip inst.705 if (!LeafFound)706 continue;707 708 // First add !memprof metadata from allocation info, if we found the709 // instruction's leaf location in that map, and if the rest of the710 // instruction's locations match the prefix Frame locations on an711 // allocation context with the same leaf.712 if (AllocInfoIter != LocHashToAllocInfo.end() &&713 // Only consider allocations which support hinting.714 isAllocationWithHotColdVariant(CI->getCalledFunction(), TLI))715 handleAllocSite(I, CI, InlinedCallStack, Ctx, ORE, MaxColdSize,716 AllocInfoIter->second, FullStackIdToAllocMatchInfo);717 else if (CallSitesIter != LocHashToCallSites.end())718 // Otherwise, add callsite metadata. If we reach here then we found the719 // instruction's leaf location in the callsites map and not the720 // allocation map.721 handleCallSite(I, CalledFunction, InlinedCallStack,722 CallSitesIter->second, M, MatchedCallSites);723 }724 }725}726 727MemProfUsePass::MemProfUsePass(std::string MemoryProfileFile,728 IntrusiveRefCntPtr<vfs::FileSystem> FS)729 : MemoryProfileFileName(MemoryProfileFile), FS(FS) {730 if (!FS)731 this->FS = vfs::getRealFileSystem();732}733 734PreservedAnalyses MemProfUsePass::run(Module &M, ModuleAnalysisManager &AM) {735 // Return immediately if the module doesn't contain any function or global736 // variables.737 if (M.empty() && M.globals().empty())738 return PreservedAnalyses::all();739 740 LLVM_DEBUG(dbgs() << "Read in memory profile:\n");741 auto &Ctx = M.getContext();742 auto ReaderOrErr = IndexedInstrProfReader::create(MemoryProfileFileName, *FS);743 if (Error E = ReaderOrErr.takeError()) {744 handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) {745 Ctx.diagnose(746 DiagnosticInfoPGOProfile(MemoryProfileFileName.data(), EI.message()));747 });748 return PreservedAnalyses::all();749 }750 751 std::unique_ptr<IndexedInstrProfReader> MemProfReader =752 std::move(ReaderOrErr.get());753 if (!MemProfReader) {754 Ctx.diagnose(DiagnosticInfoPGOProfile(755 MemoryProfileFileName.data(), StringRef("Cannot get MemProfReader")));756 return PreservedAnalyses::all();757 }758 759 if (!MemProfReader->hasMemoryProfile()) {760 Ctx.diagnose(DiagnosticInfoPGOProfile(MemoryProfileFileName.data(),761 "Not a memory profile"));762 return PreservedAnalyses::all();763 }764 765 const bool Changed =766 annotateGlobalVariables(M, MemProfReader->getDataAccessProfileData());767 768 // If the module doesn't contain any function, return after we process all769 // global variables.770 if (M.empty())771 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();772 773 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();774 775 TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(*M.begin());776 DenseMap<uint64_t, LocToLocMap> UndriftMaps;777 if (SalvageStaleProfile)778 UndriftMaps = computeUndriftMap(M, MemProfReader.get(), TLI);779 780 // Map from the stack hash and matched frame count of each allocation context781 // in the function profiles to the total profiled size (bytes) and allocation782 // type.783 std::map<std::pair<uint64_t, unsigned>, AllocMatchInfo>784 FullStackIdToAllocMatchInfo;785 786 // Set of the matched call sites, each expressed as a sequence of an inline787 // call stack.788 std::set<std::vector<uint64_t>> MatchedCallSites;789 790 uint64_t MaxColdSize = 0;791 if (auto *MemProfSum = MemProfReader->getMemProfSummary())792 MaxColdSize = MemProfSum->getMaxColdTotalSize();793 794 for (auto &F : M) {795 if (F.isDeclaration())796 continue;797 798 const TargetLibraryInfo &TLI = FAM.getResult<TargetLibraryAnalysis>(F);799 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(F);800 readMemprof(M, F, MemProfReader.get(), TLI, FullStackIdToAllocMatchInfo,801 MatchedCallSites, UndriftMaps, ORE, MaxColdSize);802 }803 804 if (ClPrintMemProfMatchInfo) {805 for (const auto &[IdLengthPair, Info] : FullStackIdToAllocMatchInfo) {806 auto [Id, Length] = IdLengthPair;807 errs() << "MemProf " << getAllocTypeAttributeString(Info.AllocType)808 << " context with id " << Id << " has total profiled size "809 << Info.TotalSize << " is matched with " << Length << " frames\n";810 }811 812 for (const auto &CallStack : MatchedCallSites) {813 errs() << "MemProf callsite match for inline call stack";814 for (uint64_t StackId : CallStack)815 errs() << " " << StackId;816 errs() << "\n";817 }818 }819 820 return PreservedAnalyses::none();821}822 823bool MemProfUsePass::annotateGlobalVariables(824 Module &M, const memprof::DataAccessProfData *DataAccessProf) {825 if (!AnnotateStaticDataSectionPrefix || M.globals().empty())826 return false;827 828 if (!DataAccessProf) {829 M.addModuleFlag(Module::Warning, "EnableDataAccessProf", 0U);830 M.getContext().diagnose(DiagnosticInfoPGOProfile(831 MemoryProfileFileName.data(),832 StringRef("Data access profiles not found in memprof. Ignore "833 "-memprof-annotate-static-data-prefix."),834 DS_Warning));835 return false;836 }837 M.addModuleFlag(Module::Warning, "EnableDataAccessProf", 1U);838 839 bool Changed = false;840 // Iterate all global variables in the module and annotate them based on841 // data access profiles. Note it's up to the linker to decide how to map input842 // sections to output sections, and one conservative practice is to map843 // unlikely-prefixed ones to unlikely output section, and map the rest844 // (hot-prefixed or prefix-less) to the canonical output section.845 for (GlobalVariable &GVar : M.globals()) {846 assert(!GVar.getSectionPrefix().has_value() &&847 "GVar shouldn't have section prefix yet");848 auto Kind = llvm::memprof::getAnnotationKind(GVar);849 if (Kind != llvm::memprof::AnnotationKind::AnnotationOK) {850 HandleUnsupportedAnnotationKinds(GVar, Kind);851 continue;852 }853 854 StringRef Name = GVar.getName();855 // Skip string literals as their mangled names don't stay stable across856 // binary releases.857 // TODO: Track string content hash in the profiles and compute it inside the858 // compiler to categeorize the hotness string literals.859 if (Name.starts_with(".str")) {860 LLVM_DEBUG(dbgs() << "Skip annotating string literal " << Name << "\n");861 continue;862 }863 864 // DataAccessProfRecord's get* methods will canonicalize the name under the865 // hood before looking it up, so optimizer doesn't need to do it.866 std::optional<DataAccessProfRecord> Record =867 DataAccessProf->getProfileRecord(Name);868 // Annotate a global variable as hot if it has non-zero sampled count, and869 // annotate it as cold if it's seen in the profiled binary870 // file but doesn't have any access sample.871 // For logging, optimization remark emitter requires a llvm::Function, but872 // it's not well defined how to associate a global variable with a function.873 // So we just print out the static data section prefix in LLVM_DEBUG.874 if (Record && Record->AccessCount > 0) {875 ++NumOfMemProfHotGlobalVars;876 Changed |= GVar.setSectionPrefix("hot");877 LLVM_DEBUG(dbgs() << "Global variable " << Name878 << " is annotated as hot\n");879 } else if (DataAccessProf->isKnownColdSymbol(Name)) {880 ++NumOfMemProfColdGlobalVars;881 Changed |= GVar.setSectionPrefix("unlikely");882 Changed = true;883 LLVM_DEBUG(dbgs() << "Global variable " << Name884 << " is annotated as unlikely\n");885 } else {886 ++NumOfMemProfUnknownGlobalVars;887 LLVM_DEBUG(dbgs() << "Global variable " << Name << " is not annotated\n");888 }889 }890 891 return Changed;892}893