173 lines · cpp
1//===-- AMDGPUAlwaysInlinePass.cpp - Promote Allocas ----------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file10/// This pass marks all internal functions as always_inline and creates11/// duplicates of all other functions and marks the duplicates as always_inline.12//13//===----------------------------------------------------------------------===//14 15#include "AMDGPU.h"16#include "AMDGPUTargetMachine.h"17#include "Utils/AMDGPUBaseInfo.h"18#include "llvm/CodeGen/CommandFlags.h"19#include "llvm/IR/Module.h"20#include "llvm/Pass.h"21#include "llvm/Support/CommandLine.h"22 23using namespace llvm;24 25namespace {26 27static cl::opt<bool> StressCalls(28 "amdgpu-stress-function-calls",29 cl::Hidden,30 cl::desc("Force all functions to be noinline"),31 cl::init(false));32 33class AMDGPUAlwaysInline : public ModulePass {34 bool GlobalOpt;35 36public:37 static char ID;38 39 AMDGPUAlwaysInline(bool GlobalOpt = false) :40 ModulePass(ID), GlobalOpt(GlobalOpt) { }41 bool runOnModule(Module &M) override;42 43 void getAnalysisUsage(AnalysisUsage &AU) const override {44 AU.setPreservesAll();45 }46};47 48} // End anonymous namespace49 50INITIALIZE_PASS(AMDGPUAlwaysInline, "amdgpu-always-inline",51 "AMDGPU Inline All Functions", false, false)52 53char AMDGPUAlwaysInline::ID = 0;54 55static void56recursivelyVisitUsers(GlobalValue &GV,57 SmallPtrSetImpl<Function *> &FuncsToAlwaysInline) {58 SmallVector<User *, 16> Stack(GV.users());59 60 SmallPtrSet<const Value *, 8> Visited;61 62 while (!Stack.empty()) {63 User *U = Stack.pop_back_val();64 if (!Visited.insert(U).second)65 continue;66 67 if (Instruction *I = dyn_cast<Instruction>(U)) {68 Function *F = I->getFunction();69 if (!AMDGPU::isEntryFunctionCC(F->getCallingConv())) {70 // FIXME: This is a horrible hack. We should always respect noinline,71 // and just let us hit the error when we can't handle this.72 //73 // Unfortunately, clang adds noinline to all functions at -O0. We have74 // to override this here until that's fixed.75 F->removeFnAttr(Attribute::NoInline);76 77 FuncsToAlwaysInline.insert(F);78 Stack.push_back(F);79 }80 81 // No need to look at further users, but we do need to inline any callers.82 continue;83 }84 85 append_range(Stack, U->users());86 }87}88 89static bool alwaysInlineImpl(Module &M, bool GlobalOpt) {90 std::vector<GlobalAlias*> AliasesToRemove;91 92 bool Changed = false;93 SmallPtrSet<Function *, 8> FuncsToAlwaysInline;94 SmallPtrSet<Function *, 8> FuncsToNoInline;95 Triple TT(M.getTargetTriple());96 97 for (GlobalAlias &A : M.aliases()) {98 if (Function* F = dyn_cast<Function>(A.getAliasee())) {99 if (TT.isAMDGCN() && A.getLinkage() != GlobalValue::InternalLinkage)100 continue;101 Changed = true;102 A.replaceAllUsesWith(F);103 AliasesToRemove.push_back(&A);104 }105 106 // FIXME: If the aliasee isn't a function, it's some kind of constant expr107 // cast that won't be inlined through.108 }109 110 if (GlobalOpt) {111 for (GlobalAlias* A : AliasesToRemove) {112 A->eraseFromParent();113 }114 }115 116 // Always force inlining of any function that uses an LDS global address. This117 // is something of a workaround because we don't have a way of supporting LDS118 // objects defined in functions. LDS is always allocated by a kernel, and it119 // is difficult to manage LDS usage if a function may be used by multiple120 // kernels.121 //122 // OpenCL doesn't allow declaring LDS in non-kernels, so in practice this123 // should only appear when IPO passes manages to move LDs defined in a kernel124 // into a single user function.125 126 for (GlobalVariable &GV : M.globals()) {127 // TODO: Region address128 unsigned AS = GV.getAddressSpace();129 if ((AS == AMDGPUAS::REGION_ADDRESS) ||130 (AS == AMDGPUAS::LOCAL_ADDRESS &&131 (!AMDGPUTargetMachine::EnableLowerModuleLDS)))132 recursivelyVisitUsers(GV, FuncsToAlwaysInline);133 }134 135 if (!AMDGPUTargetMachine::EnableFunctionCalls || StressCalls) {136 auto IncompatAttr137 = StressCalls ? Attribute::AlwaysInline : Attribute::NoInline;138 139 for (Function &F : M) {140 if (!F.isDeclaration() && !F.use_empty() &&141 !F.hasFnAttribute(IncompatAttr)) {142 if (StressCalls) {143 if (!FuncsToAlwaysInline.count(&F))144 FuncsToNoInline.insert(&F);145 } else146 FuncsToAlwaysInline.insert(&F);147 }148 }149 }150 151 for (Function *F : FuncsToAlwaysInline)152 F->addFnAttr(Attribute::AlwaysInline);153 154 for (Function *F : FuncsToNoInline)155 F->addFnAttr(Attribute::NoInline);156 157 return Changed || !FuncsToAlwaysInline.empty() || !FuncsToNoInline.empty();158}159 160bool AMDGPUAlwaysInline::runOnModule(Module &M) {161 return alwaysInlineImpl(M, GlobalOpt);162}163 164ModulePass *llvm::createAMDGPUAlwaysInlinePass(bool GlobalOpt) {165 return new AMDGPUAlwaysInline(GlobalOpt);166}167 168PreservedAnalyses AMDGPUAlwaysInlinePass::run(Module &M,169 ModuleAnalysisManager &AM) {170 const bool Changed = alwaysInlineImpl(M, GlobalOpt);171 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();172}173