190 lines · cpp
1//===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements a custom inliner that handles only functions that10// are marked as "always inline".11//12//===----------------------------------------------------------------------===//13 14#include "llvm/Transforms/IPO/AlwaysInliner.h"15#include "llvm/ADT/SetVector.h"16#include "llvm/Analysis/AliasAnalysis.h"17#include "llvm/Analysis/AssumptionCache.h"18#include "llvm/Analysis/InlineAdvisor.h"19#include "llvm/Analysis/InlineCost.h"20#include "llvm/Analysis/OptimizationRemarkEmitter.h"21#include "llvm/Analysis/ProfileSummaryInfo.h"22#include "llvm/IR/Module.h"23#include "llvm/InitializePasses.h"24#include "llvm/Transforms/Utils/Cloning.h"25#include "llvm/Transforms/Utils/ModuleUtils.h"26 27using namespace llvm;28 29#define DEBUG_TYPE "inline"30 31namespace {32 33bool AlwaysInlineImpl(34 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI,35 FunctionAnalysisManager *FAM,36 function_ref<AssumptionCache &(Function &)> GetAssumptionCache,37 function_ref<AAResults &(Function &)> GetAAR) {38 SmallSetVector<CallBase *, 16> Calls;39 bool Changed = false;40 SmallVector<Function *, 16> InlinedComdatFunctions;41 42 for (Function &F : make_early_inc_range(M)) {43 if (F.isPresplitCoroutine())44 continue;45 46 if (F.isDeclaration() || !isInlineViable(F).isSuccess())47 continue;48 49 Calls.clear();50 51 for (User *U : F.users())52 if (auto *CB = dyn_cast<CallBase>(U))53 if (CB->getCalledFunction() == &F &&54 CB->hasFnAttr(Attribute::AlwaysInline) &&55 !CB->getAttributes().hasFnAttr(Attribute::NoInline))56 Calls.insert(CB);57 58 for (CallBase *CB : Calls) {59 Function *Caller = CB->getCaller();60 OptimizationRemarkEmitter ORE(Caller);61 DebugLoc DLoc = CB->getDebugLoc();62 BasicBlock *Block = CB->getParent();63 64 InlineFunctionInfo IFI(GetAssumptionCache, &PSI, nullptr, nullptr);65 InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true,66 &GetAAR(F), InsertLifetime);67 if (!Res.isSuccess()) {68 ORE.emit([&]() {69 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)70 << "'" << ore::NV("Callee", &F) << "' is not inlined into '"71 << ore::NV("Caller", Caller)72 << "': " << ore::NV("Reason", Res.getFailureReason());73 });74 continue;75 }76 77 emitInlinedIntoBasedOnCost(78 ORE, DLoc, Block, F, *Caller,79 InlineCost::getAlways("always inline attribute"),80 /*ForProfileContext=*/false, DEBUG_TYPE);81 82 Changed = true;83 if (FAM)84 FAM->invalidate(*Caller, PreservedAnalyses::none());85 }86 87 F.removeDeadConstantUsers();88 if (F.hasFnAttribute(Attribute::AlwaysInline) && F.isDefTriviallyDead()) {89 // Remember to try and delete this function afterward. This allows to call90 // filterDeadComdatFunctions() only once.91 if (F.hasComdat()) {92 InlinedComdatFunctions.push_back(&F);93 } else {94 if (FAM)95 FAM->clear(F, F.getName());96 M.getFunctionList().erase(F);97 Changed = true;98 }99 }100 }101 102 if (!InlinedComdatFunctions.empty()) {103 // Now we just have the comdat functions. Filter out the ones whose comdats104 // are not actually dead.105 filterDeadComdatFunctions(InlinedComdatFunctions);106 // The remaining functions are actually dead.107 for (Function *F : InlinedComdatFunctions) {108 if (FAM)109 FAM->clear(*F, F->getName());110 M.getFunctionList().erase(F);111 Changed = true;112 }113 }114 115 return Changed;116}117 118struct AlwaysInlinerLegacyPass : public ModulePass {119 bool InsertLifetime;120 121 AlwaysInlinerLegacyPass()122 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {}123 124 AlwaysInlinerLegacyPass(bool InsertLifetime)125 : ModulePass(ID), InsertLifetime(InsertLifetime) {126 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());127 }128 129 /// Main run interface method.130 bool runOnModule(Module &M) override {131 132 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();133 auto GetAAR = [&](Function &F) -> AAResults & {134 return getAnalysis<AAResultsWrapperPass>(F).getAAResults();135 };136 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {137 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);138 };139 140 return AlwaysInlineImpl(M, InsertLifetime, PSI, /*FAM=*/nullptr,141 GetAssumptionCache, GetAAR);142 }143 144 static char ID; // Pass identification, replacement for typeid145 146 void getAnalysisUsage(AnalysisUsage &AU) const override {147 AU.addRequired<AssumptionCacheTracker>();148 AU.addRequired<AAResultsWrapperPass>();149 AU.addRequired<ProfileSummaryInfoWrapperPass>();150 }151};152 153} // namespace154 155char AlwaysInlinerLegacyPass::ID = 0;156INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",157 "Inliner for always_inline functions", false, false)158INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)159INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)160INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)161INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",162 "Inliner for always_inline functions", false, false)163 164Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {165 return new AlwaysInlinerLegacyPass(InsertLifetime);166}167 168PreservedAnalyses AlwaysInlinerPass::run(Module &M,169 ModuleAnalysisManager &MAM) {170 FunctionAnalysisManager &FAM =171 MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();172 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {173 return FAM.getResult<AssumptionAnalysis>(F);174 };175 auto GetAAR = [&](Function &F) -> AAResults & {176 return FAM.getResult<AAManager>(F);177 };178 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M);179 180 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, &FAM,181 GetAssumptionCache, GetAAR);182 if (!Changed)183 return PreservedAnalyses::all();184 185 PreservedAnalyses PA;186 // We have already invalidated all analyses on modified functions.187 PA.preserveSet<AllAnalysesOn<Function>>();188 return PA;189}190