692 lines · cpp
1//===- InlineAdvisor.cpp - analysis pass implementation -------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and10// related types.11//12//===----------------------------------------------------------------------===//13 14#include "llvm/Analysis/InlineAdvisor.h"15#include "llvm/ADT/Statistic.h"16#include "llvm/ADT/StringExtras.h"17#include "llvm/Analysis/AssumptionCache.h"18#include "llvm/Analysis/EphemeralValuesCache.h"19#include "llvm/Analysis/IR2Vec.h"20#include "llvm/Analysis/InlineCost.h"21#include "llvm/Analysis/OptimizationRemarkEmitter.h"22#include "llvm/Analysis/ProfileSummaryInfo.h"23#include "llvm/Analysis/ReplayInlineAdvisor.h"24#include "llvm/Analysis/TargetLibraryInfo.h"25#include "llvm/Analysis/TargetTransformInfo.h"26#include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"27#include "llvm/IR/DebugInfoMetadata.h"28#include "llvm/IR/Module.h"29#include "llvm/IR/PassManager.h"30#include "llvm/Support/CommandLine.h"31#include "llvm/Support/raw_ostream.h"32 33using namespace llvm;34#define DEBUG_TYPE "inline"35#ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL36#define LLVM_HAVE_TF_AOT37#endif38 39// This weirdly named statistic tracks the number of times that, when attempting40// to inline a function A into B, we analyze the callers of B in order to see41// if those would be more profitable and blocked inline steps.42STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");43 44/// Flag to add inline messages as callsite attributes 'inline-remark'.45static cl::opt<bool>46 InlineRemarkAttribute("inline-remark-attribute", cl::init(false),47 cl::Hidden,48 cl::desc("Enable adding inline-remark attribute to"49 " callsites processed by inliner but decided"50 " to be not inlined"));51 52static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false),53 cl::Hidden,54 cl::desc("Enable deferred inlining"));55 56// An integer used to limit the cost of inline deferral. The default negative57// number tells shouldBeDeferred to only take the secondary cost into account.58static cl::opt<int>59 InlineDeferralScale("inline-deferral-scale",60 cl::desc("Scale to limit the cost of inline deferral"),61 cl::init(2), cl::Hidden);62 63static cl::opt<bool>64 AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false),65 cl::desc("If true, annotate inline advisor remarks "66 "with LTO and pass information."));67 68// This flag is used to enable IR2Vec embeddings in the ML inliner; Only valid69// with ML inliner. The vocab file is used to initialize the embeddings.70static cl::opt<std::string> IR2VecVocabFile(71 "ml-inliner-ir2vec-vocab-file", cl::Hidden,72 cl::desc("Vocab file for IR2Vec; Setting this enables "73 "configuring the model to use IR2Vec embeddings."));74 75namespace llvm {76extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;77} // namespace llvm78 79namespace {80using namespace llvm::ore;81class MandatoryInlineAdvice : public InlineAdvice {82public:83 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,84 OptimizationRemarkEmitter &ORE,85 bool IsInliningMandatory)86 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}87 88private:89 void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); }90 91 void recordInliningImpl() override {92 if (IsInliningRecommended)93 emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended,94 [&](OptimizationRemark &Remark) {95 Remark << ": always inline attribute";96 });97 }98 99 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {100 if (IsInliningRecommended)101 ORE.emit([&]() {102 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),103 "NotInlined", DLoc, Block)104 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"105 << NV("Caller", Caller)106 << "': " << NV("Reason", Result.getFailureReason());107 });108 }109 110 void recordUnattemptedInliningImpl() override {111 assert(!IsInliningRecommended && "Expected to attempt inlining");112 }113};114} // namespace115 116void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(117 const InlineResult &Result) {118 using namespace ore;119 llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +120 "; " + inlineCostStr(*OIC));121 ORE.emit([&]() {122 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),123 "NotInlined", DLoc, Block)124 << "'" << NV("Callee", Callee) << "' is not inlined into '"125 << NV("Caller", Caller)126 << "': " << NV("Reason", Result.getFailureReason());127 });128}129 130void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {131 if (EmitRemarks)132 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC,133 /* ForProfileContext= */ false,134 Advisor->getAnnotatedInlinePassName());135}136 137void DefaultInlineAdvice::recordInliningImpl() {138 if (EmitRemarks)139 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC,140 /* ForProfileContext= */ false,141 Advisor->getAnnotatedInlinePassName());142}143 144std::optional<llvm::InlineCost> static getDefaultInlineAdvice(145 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {146 Function &Caller = *CB.getCaller();147 ProfileSummaryInfo *PSI =148 FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller)149 .getCachedResult<ProfileSummaryAnalysis>(150 *CB.getParent()->getParent()->getParent());151 152 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);153 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {154 return FAM.getResult<AssumptionAnalysis>(F);155 };156 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {157 return FAM.getResult<BlockFrequencyAnalysis>(F);158 };159 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {160 return FAM.getResult<TargetLibraryAnalysis>(F);161 };162 auto GetEphValuesCache =163 [&](Function &F) -> EphemeralValuesAnalysis::Result & {164 return FAM.getResult<EphemeralValuesAnalysis>(F);165 };166 167 Function &Callee = *CB.getCalledFunction();168 auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);169 auto GetInlineCost = [&](CallBase &CB) {170 bool RemarksEnabled =171 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(172 DEBUG_TYPE);173 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,174 GetBFI, PSI, RemarksEnabled ? &ORE : nullptr,175 GetEphValuesCache);176 };177 return llvm::shouldInline(178 CB, CalleeTTI, GetInlineCost, ORE,179 Params.EnableDeferral.value_or(EnableInlineDeferral));180}181 182std::unique_ptr<InlineAdvice>183DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {184 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);185 return std::make_unique<DefaultInlineAdvice>(186 this, CB, OIC,187 FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller()));188}189 190InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,191 OptimizationRemarkEmitter &ORE,192 bool IsInliningRecommended)193 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),194 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),195 IsInliningRecommended(IsInliningRecommended) {}196 197void InlineAdvice::recordInlineStatsIfNeeded() {198 if (Advisor->ImportedFunctionsStats)199 Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee);200}201 202void InlineAdvice::recordInlining() {203 markRecorded();204 recordInlineStatsIfNeeded();205 recordInliningImpl();206}207 208void InlineAdvice::recordInliningWithCalleeDeleted() {209 markRecorded();210 recordInlineStatsIfNeeded();211 recordInliningWithCalleeDeletedImpl();212}213 214AnalysisKey InlineAdvisorAnalysis::Key;215AnalysisKey PluginInlineAdvisorAnalysis::Key;216 217bool InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(218 Module &M, ModuleAnalysisManager &MAM) {219 if (!IR2VecVocabFile.empty()) {220 auto &IR2VecVocabResult = MAM.getResult<IR2VecVocabAnalysis>(M);221 if (!IR2VecVocabResult.isValid()) {222 M.getContext().emitError("Failed to load IR2Vec vocabulary");223 return false;224 }225 }226 // No vocab file specified is OK; We just don't use IR2Vec227 // embeddings.228 return true;229}230 231bool InlineAdvisorAnalysis::Result::tryCreate(232 InlineParams Params, InliningAdvisorMode Mode,233 const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {234 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();235 if (MAM.isPassRegistered<PluginInlineAdvisorAnalysis>()) {236 auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(M);237 Advisor.reset(DA.Factory(M, FAM, Params, IC));238 return !!Advisor;239 }240 auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) {241 auto OIC = getDefaultInlineAdvice(CB, FAM, Params);242 return OIC.has_value();243 };244 switch (Mode) {245 case InliningAdvisorMode::Default:246 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");247 Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC));248 // Restrict replay to default advisor, ML advisors are stateful so249 // replay will need augmentations to interleave with them correctly.250 if (!ReplaySettings.ReplayFile.empty()) {251 Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(),252 std::move(Advisor), ReplaySettings,253 /* EmitRemarks =*/true, IC);254 }255 break;256 // Run IR2VecVocabAnalysis once per module to get the vocabulary.257 // We run it here because it is immutable and we want to avoid running it258 // multiple times.259 case InliningAdvisorMode::Development:260#ifdef LLVM_HAVE_TFLITE261 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");262 if (!InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(M, MAM))263 return false;264 Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);265#endif266 break;267 case InliningAdvisorMode::Release:268 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");269 if (!InlineAdvisorAnalysis::initializeIR2VecVocabIfRequested(M, MAM))270 return false;271 Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);272 break;273 }274 275 return !!Advisor;276}277 278/// Return true if inlining of CB can block the caller from being279/// inlined which is proved to be more beneficial. \p IC is the280/// estimated inline cost associated with callsite \p CB.281/// \p TotalSecondaryCost will be set to the estimated cost of inlining the282/// caller if \p CB is suppressed for inlining.283static bool284shouldBeDeferred(Function *Caller, TargetTransformInfo &CalleeTTI,285 InlineCost IC, int &TotalSecondaryCost,286 function_ref<InlineCost(CallBase &CB)> GetInlineCost) {287 // For now we only handle local or inline functions.288 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())289 return false;290 // If the cost of inlining CB is non-positive, it is not going to prevent the291 // caller from being inlined into its callers and hence we don't need to292 // defer.293 if (IC.getCost() <= 0)294 return false;295 // Try to detect the case where the current inlining candidate caller (call296 // it B) is a static or linkonce-ODR function and is an inlining candidate297 // elsewhere, and the current candidate callee (call it C) is large enough298 // that inlining it into B would make B too big to inline later. In these299 // circumstances it may be best not to inline C into B, but to inline B into300 // its callers.301 //302 // This only applies to static and linkonce-ODR functions because those are303 // expected to be available for inlining in the translation units where they304 // are used. Thus we will always have the opportunity to make local inlining305 // decisions. Importantly the linkonce-ODR linkage covers inline functions306 // and templates in C++.307 //308 // FIXME: All of this logic should be sunk into getInlineCost. It relies on309 // the internal implementation of the inline cost metrics rather than310 // treating them as truly abstract units etc.311 TotalSecondaryCost = 0;312 // The candidate cost to be imposed upon the current function.313 int CandidateCost = IC.getCost() - 1;314 // If the caller has local linkage and can be inlined to all its callers, we315 // can apply a huge negative bonus to TotalSecondaryCost.316 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();317 // This bool tracks what happens if we DO inline C into B.318 bool InliningPreventsSomeOuterInline = false;319 unsigned NumCallerUsers = 0;320 for (User *U : Caller->users()) {321 CallBase *CS2 = dyn_cast<CallBase>(U);322 323 // If this isn't a call to Caller (it could be some other sort324 // of reference) skip it. Such references will prevent the caller325 // from being removed.326 if (!CS2 || CS2->getCalledFunction() != Caller) {327 ApplyLastCallBonus = false;328 continue;329 }330 331 InlineCost IC2 = GetInlineCost(*CS2);332 ++NumCallerCallersAnalyzed;333 if (!IC2) {334 ApplyLastCallBonus = false;335 continue;336 }337 if (IC2.isAlways())338 continue;339 340 // See if inlining of the original callsite would erase the cost delta of341 // this callsite. We subtract off the penalty for the call instruction,342 // which we would be deleting.343 if (IC2.getCostDelta() <= CandidateCost) {344 InliningPreventsSomeOuterInline = true;345 TotalSecondaryCost += IC2.getCost();346 NumCallerUsers++;347 }348 }349 350 if (!InliningPreventsSomeOuterInline)351 return false;352 353 // If all outer calls to Caller would get inlined, the cost for the last354 // one is set very low by getInlineCost, in anticipation that Caller will355 // be removed entirely. We did not account for this above unless there356 // is only one caller of Caller.357 if (ApplyLastCallBonus)358 TotalSecondaryCost -= CalleeTTI.getInliningLastCallToStaticBonus();359 360 // If InlineDeferralScale is negative, then ignore the cost of primary361 // inlining -- IC.getCost() multiplied by the number of callers to Caller.362 if (InlineDeferralScale < 0)363 return TotalSecondaryCost < IC.getCost();364 365 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;366 int Allowance = IC.getCost() * InlineDeferralScale;367 return TotalCost < Allowance;368}369 370namespace llvm {371static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {372 return R << Arg.Val;373}374 375template <class RemarkT>376decltype(auto) operator<<(RemarkT &&R, const InlineCost &IC) {377 using namespace ore;378 if (IC.isAlways()) {379 R << "(cost=always)";380 } else if (IC.isNever()) {381 R << "(cost=never)";382 } else {383 R << "(cost=" << ore::NV("Cost", IC.getCost())384 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";385 }386 if (const char *Reason = IC.getReason())387 R << ": " << ore::NV("Reason", Reason);388 return std::forward<RemarkT>(R);389}390} // namespace llvm391 392std::string llvm::inlineCostStr(const InlineCost &IC) {393 std::string Buffer;394 raw_string_ostream Remark(Buffer);395 Remark << IC;396 return Remark.str();397}398 399void llvm::setInlineRemark(CallBase &CB, StringRef Message) {400 if (!InlineRemarkAttribute)401 return;402 403 Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message);404 CB.addFnAttr(Attr);405}406 407/// Return the cost only if the inliner should attempt to inline at the given408/// CallSite. If we return the cost, we will emit an optimisation remark later409/// using that cost, so we won't do so from this function. Return std::nullopt410/// if inlining should not be attempted.411std::optional<InlineCost>412llvm::shouldInline(CallBase &CB, TargetTransformInfo &CalleeTTI,413 function_ref<InlineCost(CallBase &CB)> GetInlineCost,414 OptimizationRemarkEmitter &ORE, bool EnableDeferral) {415 using namespace ore;416 417 InlineCost IC = GetInlineCost(CB);418 Instruction *Call = &CB;419 Function *Callee = CB.getCalledFunction();420 Function *Caller = CB.getCaller();421 422 if (IC.isAlways()) {423 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC)424 << ", Call: " << CB << "\n");425 return IC;426 }427 428 if (!IC) {429 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC)430 << ", Call: " << CB << "\n");431 if (IC.isNever()) {432 ORE.emit([&]() {433 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)434 << "'" << NV("Callee", Callee) << "' not inlined into '"435 << NV("Caller", Caller)436 << "' because it should never be inlined " << IC;437 });438 } else {439 ORE.emit([&]() {440 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)441 << "'" << NV("Callee", Callee) << "' not inlined into '"442 << NV("Caller", Caller) << "' because too costly to inline "443 << IC;444 });445 }446 setInlineRemark(CB, inlineCostStr(IC));447 return std::nullopt;448 }449 450 int TotalSecondaryCost = 0;451 if (EnableDeferral && shouldBeDeferred(Caller, CalleeTTI, IC,452 TotalSecondaryCost, GetInlineCost)) {453 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB454 << " Cost = " << IC.getCost()455 << ", outer Cost = " << TotalSecondaryCost << '\n');456 ORE.emit([&]() {457 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",458 Call)459 << "Not inlining. Cost of inlining '" << NV("Callee", Callee)460 << "' increases the cost of inlining '" << NV("Caller", Caller)461 << "' in other contexts";462 });463 setInlineRemark(CB, "deferred");464 return std::nullopt;465 }466 467 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB468 << '\n');469 return IC;470}471 472std::string llvm::formatCallSiteLocation(DebugLoc DLoc,473 const CallSiteFormat &Format) {474 std::string Buffer;475 raw_string_ostream CallSiteLoc(Buffer);476 bool First = true;477 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {478 if (!First)479 CallSiteLoc << " @ ";480 // Note that negative line offset is actually possible, but we use481 // unsigned int to match line offset representation in remarks so482 // it's directly consumable by relay advisor.483 uint32_t Offset =484 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();485 uint32_t Discriminator = DIL->getBaseDiscriminator();486 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();487 if (Name.empty())488 Name = DIL->getScope()->getSubprogram()->getName();489 CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset);490 if (Format.outputColumn())491 CallSiteLoc << ":" << llvm::utostr(DIL->getColumn());492 if (Format.outputDiscriminator() && Discriminator)493 CallSiteLoc << "." << llvm::utostr(Discriminator);494 First = false;495 }496 497 return CallSiteLoc.str();498}499 500void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) {501 if (!DLoc) {502 return;503 }504 505 bool First = true;506 Remark << " at callsite ";507 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {508 if (!First)509 Remark << " @ ";510 unsigned int Offset = DIL->getLine();511 Offset -= DIL->getScope()->getSubprogram()->getLine();512 unsigned int Discriminator = DIL->getBaseDiscriminator();513 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();514 if (Name.empty())515 Name = DIL->getScope()->getSubprogram()->getName();516 Remark << Name << ":" << ore::NV("Line", Offset) << ":"517 << ore::NV("Column", DIL->getColumn());518 if (Discriminator)519 Remark << "." << ore::NV("Disc", Discriminator);520 First = false;521 }522 523 Remark << ";";524}525 526void llvm::emitInlinedInto(527 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,528 const Function &Callee, const Function &Caller, bool AlwaysInline,529 function_ref<void(OptimizationRemark &)> ExtraContext,530 const char *PassName) {531 ORE.emit([&]() {532 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";533 OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName,534 DLoc, Block);535 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"536 << ore::NV("Caller", &Caller) << "'";537 if (ExtraContext)538 ExtraContext(Remark);539 addLocationToRemarks(Remark, DLoc);540 return Remark;541 });542}543 544void llvm::emitInlinedIntoBasedOnCost(545 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,546 const Function &Callee, const Function &Caller, const InlineCost &IC,547 bool ForProfileContext, const char *PassName) {548 llvm::emitInlinedInto(549 ORE, DLoc, Block, Callee, Caller, IC.isAlways(),550 [&](OptimizationRemark &Remark) {551 if (ForProfileContext)552 Remark << " to match profiling context";553 Remark << " with " << IC;554 },555 PassName);556}557 558InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM,559 std::optional<InlineContext> IC)560 : M(M), FAM(FAM), IC(IC),561 AnnotatedInlinePassName((IC && AnnotateInlinePhase)562 ? llvm::AnnotateInlinePassName(*IC)563 : DEBUG_TYPE) {564 if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {565 ImportedFunctionsStats =566 std::make_unique<ImportedFunctionsInliningStatistics>();567 ImportedFunctionsStats->setModuleInfo(M);568 }569}570 571InlineAdvisor::~InlineAdvisor() {572 if (ImportedFunctionsStats) {573 assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);574 ImportedFunctionsStats->dump(InlinerFunctionImportStats ==575 InlinerFunctionImportStatsOpts::Verbose);576 }577}578 579std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,580 bool Advice) {581 return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB),582 Advice);583}584 585static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {586 switch (LTOPhase) {587 case (ThinOrFullLTOPhase::None):588 return "main";589 case (ThinOrFullLTOPhase::ThinLTOPreLink):590 case (ThinOrFullLTOPhase::FullLTOPreLink):591 return "prelink";592 case (ThinOrFullLTOPhase::ThinLTOPostLink):593 case (ThinOrFullLTOPhase::FullLTOPostLink):594 return "postlink";595 }596 llvm_unreachable("unreachable");597}598 599static inline const char *getInlineAdvisorContext(InlinePass IP) {600 switch (IP) {601 case (InlinePass::AlwaysInliner):602 return "always-inline";603 case (InlinePass::CGSCCInliner):604 return "cgscc-inline";605 case (InlinePass::EarlyInliner):606 return "early-inline";607 case (InlinePass::MLInliner):608 return "ml-inline";609 case (InlinePass::ModuleInliner):610 return "module-inline";611 case (InlinePass::ReplayCGSCCInliner):612 return "replay-cgscc-inline";613 case (InlinePass::ReplaySampleProfileInliner):614 return "replay-sample-profile-inline";615 case (InlinePass::SampleProfileInliner):616 return "sample-profile-inline";617 }618 619 llvm_unreachable("unreachable");620}621 622std::string llvm::AnnotateInlinePassName(InlineContext IC) {623 return std::string(getLTOPhase(IC.LTOPhase)) + "-" +624 std::string(getInlineAdvisorContext(IC.Pass));625}626 627InlineAdvisor::MandatoryInliningKind628InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM,629 OptimizationRemarkEmitter &ORE) {630 auto &Callee = *CB.getCalledFunction();631 632 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {633 return FAM.getResult<TargetLibraryAnalysis>(F);634 };635 636 auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);637 638 auto TrivialDecision =639 llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);640 641 if (TrivialDecision) {642 if (TrivialDecision->isSuccess())643 return MandatoryInliningKind::Always;644 else645 return MandatoryInliningKind::Never;646 }647 return MandatoryInliningKind::NotMandatory;648}649 650std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,651 bool MandatoryOnly) {652 if (!MandatoryOnly)653 return getAdviceImpl(CB);654 bool Advice = CB.getCaller() != CB.getCalledFunction() &&655 MandatoryInliningKind::Always ==656 getMandatoryKind(CB, FAM, getCallerORE(CB));657 return getMandatoryAdvice(CB, Advice);658}659 660OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) {661 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller());662}663 664PreservedAnalyses665InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) {666 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);667 if (!IA)668 OS << "No Inline Advisor\n";669 else670 IA->getAdvisor()->print(OS);671 return PreservedAnalyses::all();672}673 674PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run(675 LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG,676 CGSCCUpdateResult &UR) {677 const auto &MAMProxy =678 AM.getResult<ModuleAnalysisManagerCGSCCProxy>(InitialC, CG);679 680 if (InitialC.size() == 0) {681 OS << "SCC is empty!\n";682 return PreservedAnalyses::all();683 }684 Module &M = *InitialC.begin()->getFunction().getParent();685 const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M);686 if (!IA)687 OS << "No Inline Advisor\n";688 else689 IA->getAdvisor()->print(OS);690 return PreservedAnalyses::all();691}692