280 lines · cpp
1//===- ProfileSummaryInfo.cpp - Global profile summary information --------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file contains a pass that provides access to the global profile summary10// information.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/Analysis/ProfileSummaryInfo.h"15#include "llvm/Analysis/BlockFrequencyInfo.h"16#include "llvm/IR/BasicBlock.h"17#include "llvm/IR/Instructions.h"18#include "llvm/IR/Module.h"19#include "llvm/IR/ProfileSummary.h"20#include "llvm/InitializePasses.h"21#include "llvm/ProfileData/ProfileCommon.h"22#include "llvm/Support/CommandLine.h"23#include "llvm/Support/Compiler.h"24#include <optional>25using namespace llvm;26 27namespace llvm {28 29static cl::opt<bool> PartialProfile(30 "partial-profile", cl::Hidden, cl::init(false),31 cl::desc("Specify the current profile is used as a partial profile."));32 33LLVM_ABI cl::opt<bool> ScalePartialSampleProfileWorkingSetSize(34 "scale-partial-sample-profile-working-set-size", cl::Hidden, cl::init(true),35 cl::desc(36 "If true, scale the working set size of the partial sample profile "37 "by the partial profile ratio to reflect the size of the program "38 "being compiled."));39 40static cl::opt<double> PartialSampleProfileWorkingSetSizeScaleFactor(41 "partial-sample-profile-working-set-size-scale-factor", cl::Hidden,42 cl::init(0.008),43 cl::desc("The scale factor used to scale the working set size of the "44 "partial sample profile along with the partial profile ratio. "45 "This includes the factor of the profile counter per block "46 "and the factor to scale the working set size to use the same "47 "shared thresholds as PGO."));48 49} // end namespace llvm50 51// The profile summary metadata may be attached either by the frontend or by52// any backend passes (IR level instrumentation, for example). This method53// checks if the Summary is null and if so checks if the summary metadata is now54// available in the module and parses it to get the Summary object.55void ProfileSummaryInfo::refresh(std::unique_ptr<ProfileSummary> &&Other) {56 if (Other) {57 Summary.swap(Other);58 return;59 }60 if (hasProfileSummary())61 return;62 // First try to get context sensitive ProfileSummary.63 auto *SummaryMD = M->getProfileSummary(/* IsCS */ true);64 if (SummaryMD)65 Summary.reset(ProfileSummary::getFromMD(SummaryMD));66 67 if (!hasProfileSummary()) {68 // This will actually return PSK_Instr or PSK_Sample summary.69 SummaryMD = M->getProfileSummary(/* IsCS */ false);70 if (SummaryMD)71 Summary.reset(ProfileSummary::getFromMD(SummaryMD));72 }73 if (!hasProfileSummary())74 return;75 computeThresholds();76}77 78std::optional<uint64_t> ProfileSummaryInfo::getProfileCount(79 const CallBase &Call, BlockFrequencyInfo *BFI, bool AllowSynthetic) const {80 assert((isa<CallInst>(Call) || isa<InvokeInst>(Call)) &&81 "We can only get profile count for call/invoke instruction.");82 if (hasSampleProfile()) {83 // In sample PGO mode, check if there is a profile metadata on the84 // instruction. If it is present, determine hotness solely based on that,85 // since the sampled entry count may not be accurate. If there is no86 // annotated on the instruction, return std::nullopt.87 uint64_t TotalCount;88 if (Call.extractProfTotalWeight(TotalCount))89 return TotalCount;90 return std::nullopt;91 }92 if (BFI)93 return BFI->getBlockProfileCount(Call.getParent(), AllowSynthetic);94 return std::nullopt;95}96 97bool ProfileSummaryInfo::isFunctionHotnessUnknown(const Function &F) const {98 assert(hasPartialSampleProfile() && "Expect partial sample profile");99 return !F.getEntryCount();100}101 102/// Returns true if the function's entry is a cold. If it returns false, it103/// either means it is not cold or it is unknown whether it is cold or not (for104/// example, no profile data is available).105bool ProfileSummaryInfo::isFunctionEntryCold(const Function *F) const {106 if (!F)107 return false;108 if (F->hasFnAttribute(Attribute::Cold))109 return true;110 if (!hasProfileSummary())111 return false;112 auto FunctionCount = F->getEntryCount();113 // FIXME: The heuristic used below for determining coldness is based on114 // preliminary SPEC tuning for inliner. This will eventually be a115 // convenience method that calls isHotCount.116 return FunctionCount && isColdCount(FunctionCount->getCount());117}118 119/// Compute the hot and cold thresholds.120void ProfileSummaryInfo::computeThresholds() {121 auto &DetailedSummary = Summary->getDetailedSummary();122 auto &HotEntry = ProfileSummaryBuilder::getEntryForPercentile(123 DetailedSummary, ProfileSummaryCutoffHot);124 HotCountThreshold =125 ProfileSummaryBuilder::getHotCountThreshold(DetailedSummary);126 ColdCountThreshold =127 ProfileSummaryBuilder::getColdCountThreshold(DetailedSummary);128 // When the hot and cold thresholds are identical, we would classify129 // a count value as both hot and cold since we are doing an inclusive check130 // (see ::is{Hot|Cold}Count(). To avoid this undesirable overlap, ensure the131 // thresholds are distinct.132 if (HotCountThreshold == ColdCountThreshold) {133 if (ColdCountThreshold > 0)134 (*ColdCountThreshold)--;135 else136 (*HotCountThreshold)++;137 }138 assert(ColdCountThreshold < HotCountThreshold &&139 "Cold count threshold should be less than hot count threshold!");140 if (!hasPartialSampleProfile() || !ScalePartialSampleProfileWorkingSetSize) {141 HasHugeWorkingSetSize =142 HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;143 HasLargeWorkingSetSize =144 HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;145 } else {146 // Scale the working set size of the partial sample profile to reflect the147 // size of the program being compiled.148 double PartialProfileRatio = Summary->getPartialProfileRatio();149 uint64_t ScaledHotEntryNumCounts =150 static_cast<uint64_t>(HotEntry.NumCounts * PartialProfileRatio *151 PartialSampleProfileWorkingSetSizeScaleFactor);152 HasHugeWorkingSetSize =153 ScaledHotEntryNumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;154 HasLargeWorkingSetSize =155 ScaledHotEntryNumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;156 }157}158 159std::optional<uint64_t>160ProfileSummaryInfo::computeThreshold(int PercentileCutoff) const {161 if (!hasProfileSummary())162 return std::nullopt;163 auto [Iter, Inserted] = ThresholdCache.try_emplace(PercentileCutoff);164 if (!Inserted)165 return Iter->second;166 auto &DetailedSummary = Summary->getDetailedSummary();167 auto &Entry = ProfileSummaryBuilder::getEntryForPercentile(DetailedSummary,168 PercentileCutoff);169 uint64_t CountThreshold = Entry.MinCount;170 Iter->second = CountThreshold;171 return CountThreshold;172}173 174bool ProfileSummaryInfo::hasHugeWorkingSetSize() const {175 return HasHugeWorkingSetSize && *HasHugeWorkingSetSize;176}177 178bool ProfileSummaryInfo::hasLargeWorkingSetSize() const {179 return HasLargeWorkingSetSize && *HasLargeWorkingSetSize;180}181 182bool ProfileSummaryInfo::isHotCount(uint64_t C) const {183 return HotCountThreshold && C >= *HotCountThreshold;184}185 186bool ProfileSummaryInfo::isColdCount(uint64_t C) const {187 return ColdCountThreshold && C <= *ColdCountThreshold;188}189 190template <bool isHot>191bool ProfileSummaryInfo::isHotOrColdCountNthPercentile(int PercentileCutoff,192 uint64_t C) const {193 auto CountThreshold = computeThreshold(PercentileCutoff);194 if (isHot)195 return CountThreshold && C >= *CountThreshold;196 else197 return CountThreshold && C <= *CountThreshold;198}199 200bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff,201 uint64_t C) const {202 return isHotOrColdCountNthPercentile<true>(PercentileCutoff, C);203}204 205bool ProfileSummaryInfo::isColdCountNthPercentile(int PercentileCutoff,206 uint64_t C) const {207 return isHotOrColdCountNthPercentile<false>(PercentileCutoff, C);208}209 210uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() const {211 return HotCountThreshold.value_or(UINT64_MAX);212}213 214uint64_t ProfileSummaryInfo::getOrCompColdCountThreshold() const {215 return ColdCountThreshold.value_or(0);216}217 218bool ProfileSummaryInfo::isHotCallSite(const CallBase &CB,219 BlockFrequencyInfo *BFI) const {220 auto C = getProfileCount(CB, BFI);221 return C && isHotCount(*C);222}223 224bool ProfileSummaryInfo::isColdCallSite(const CallBase &CB,225 BlockFrequencyInfo *BFI) const {226 auto C = getProfileCount(CB, BFI);227 if (C)228 return isColdCount(*C);229 230 // In SamplePGO, if the caller has been sampled, and there is no profile231 // annotated on the callsite, we consider the callsite as cold.232 return hasSampleProfile() && CB.getCaller()->hasProfileData();233}234 235bool ProfileSummaryInfo::hasPartialSampleProfile() const {236 return hasProfileSummary() &&237 Summary->getKind() == ProfileSummary::PSK_Sample &&238 (PartialProfile || Summary->isPartialProfile());239}240 241INITIALIZE_PASS(ProfileSummaryInfoWrapperPass, "profile-summary-info",242 "Profile summary info", false, true)243 244ProfileSummaryInfoWrapperPass::ProfileSummaryInfoWrapperPass()245 : ImmutablePass(ID) {}246 247bool ProfileSummaryInfoWrapperPass::doInitialization(Module &M) {248 PSI.reset(new ProfileSummaryInfo(M));249 return false;250}251 252bool ProfileSummaryInfoWrapperPass::doFinalization(Module &M) {253 PSI.reset();254 return false;255}256 257AnalysisKey ProfileSummaryAnalysis::Key;258ProfileSummaryInfo ProfileSummaryAnalysis::run(Module &M,259 ModuleAnalysisManager &) {260 return ProfileSummaryInfo(M);261}262 263PreservedAnalyses ProfileSummaryPrinterPass::run(Module &M,264 ModuleAnalysisManager &AM) {265 ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);266 267 OS << "Functions in " << M.getName() << " with hot/cold annotations: \n";268 for (auto &F : M) {269 OS << F.getName();270 if (PSI.isFunctionEntryHot(&F))271 OS << " :hot entry ";272 else if (PSI.isFunctionEntryCold(&F))273 OS << " :cold entry ";274 OS << "\n";275 }276 return PreservedAnalyses::all();277}278 279char ProfileSummaryInfoWrapperPass::ID = 0;280