272 lines · cpp
1//===- StaticDataSplitter.cpp ---------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// The pass uses branch profile data to assign hotness based section qualifiers10// for the following types of static data:11// - Jump tables12// - Module-internal global variables13// - Constant pools14//15// For the original RFC of this pass please see16// https://discourse.llvm.org/t/rfc-profile-guided-static-data-partitioning/8374417 18#include "llvm/ADT/Statistic.h"19#include "llvm/Analysis/ProfileSummaryInfo.h"20#include "llvm/Analysis/StaticDataProfileInfo.h"21#include "llvm/CodeGen/MBFIWrapper.h"22#include "llvm/CodeGen/MachineBasicBlock.h"23#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"24#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"25#include "llvm/CodeGen/MachineConstantPool.h"26#include "llvm/CodeGen/MachineFunction.h"27#include "llvm/CodeGen/MachineFunctionPass.h"28#include "llvm/CodeGen/MachineJumpTableInfo.h"29#include "llvm/CodeGen/Passes.h"30#include "llvm/IR/GlobalVariable.h"31#include "llvm/InitializePasses.h"32#include "llvm/Pass.h"33#include "llvm/Target/TargetLoweringObjectFile.h"34 35using namespace llvm;36 37#define DEBUG_TYPE "static-data-splitter"38 39STATISTIC(NumHotJumpTables, "Number of hot jump tables seen.");40STATISTIC(NumColdJumpTables, "Number of cold jump tables seen.");41STATISTIC(NumUnknownJumpTables,42 "Number of jump tables with unknown hotness. They are from functions "43 "without profile information.");44 45class StaticDataSplitter : public MachineFunctionPass {46 const MachineBranchProbabilityInfo *MBPI = nullptr;47 const MachineBlockFrequencyInfo *MBFI = nullptr;48 const ProfileSummaryInfo *PSI = nullptr;49 StaticDataProfileInfo *SDPI = nullptr;50 51 // If the global value is a local linkage global variable, return it.52 // Otherwise, return nullptr.53 const GlobalVariable *getLocalLinkageGlobalVariable(const GlobalValue *GV);54 55 // Returns true if the global variable is in one of {.rodata, .bss, .data,56 // .data.rel.ro} sections.57 bool inStaticDataSection(const GlobalVariable &GV, const TargetMachine &TM);58 59 // Returns the constant if the operand refers to a global variable or constant60 // that gets lowered to static data sections. Otherwise, return nullptr.61 const Constant *getConstant(const MachineOperand &Op, const TargetMachine &TM,62 const MachineConstantPool *MCP);63 64 // Use profiles to partition static data.65 bool partitionStaticDataWithProfiles(MachineFunction &MF);66 67 // Update LLVM statistics for a machine function with profiles.68 void updateStatsWithProfiles(const MachineFunction &MF);69 70 // Update LLVM statistics for a machine function without profiles.71 void updateStatsWithoutProfiles(const MachineFunction &MF);72 73 void annotateStaticDataWithoutProfiles(const MachineFunction &MF);74 75public:76 static char ID;77 78 StaticDataSplitter() : MachineFunctionPass(ID) {79 initializeStaticDataSplitterPass(*PassRegistry::getPassRegistry());80 }81 82 StringRef getPassName() const override { return "Static Data Splitter"; }83 84 void getAnalysisUsage(AnalysisUsage &AU) const override {85 MachineFunctionPass::getAnalysisUsage(AU);86 AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();87 AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();88 AU.addRequired<ProfileSummaryInfoWrapperPass>();89 AU.addRequired<StaticDataProfileInfoWrapperPass>();90 // This pass does not modify any required analysis results except91 // StaticDataProfileInfoWrapperPass, but StaticDataProfileInfoWrapperPass92 // is made an immutable pass that it won't be re-scheduled by pass manager93 // anyway. So mark setPreservesAll() here for faster compile time.94 AU.setPreservesAll();95 }96 97 bool runOnMachineFunction(MachineFunction &MF) override;98};99 100bool StaticDataSplitter::runOnMachineFunction(MachineFunction &MF) {101 MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();102 MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();103 PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();104 105 SDPI = &getAnalysis<StaticDataProfileInfoWrapperPass>()106 .getStaticDataProfileInfo();107 108 const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI &&109 MF.getFunction().hasProfileData();110 111 if (!ProfileAvailable) {112 annotateStaticDataWithoutProfiles(MF);113 updateStatsWithoutProfiles(MF);114 return false;115 }116 117 bool Changed = partitionStaticDataWithProfiles(MF);118 119 updateStatsWithProfiles(MF);120 return Changed;121}122 123const Constant *124StaticDataSplitter::getConstant(const MachineOperand &Op,125 const TargetMachine &TM,126 const MachineConstantPool *MCP) {127 if (!Op.isGlobal() && !Op.isCPI())128 return nullptr;129 130 if (Op.isGlobal()) {131 // Find global variables with local linkage.132 const GlobalVariable *GV = getLocalLinkageGlobalVariable(Op.getGlobal());133 // Skip those not eligible for annotation or not in static data sections.134 if (!GV || !llvm::memprof::IsAnnotationOK(*GV) ||135 !inStaticDataSection(*GV, TM))136 return nullptr;137 return GV;138 }139 assert(Op.isCPI() && "Op must be constant pool index in this branch");140 int CPI = Op.getIndex();141 if (CPI == -1)142 return nullptr;143 144 assert(MCP != nullptr && "Constant pool info is not available.");145 const MachineConstantPoolEntry &CPE = MCP->getConstants()[CPI];146 147 if (CPE.isMachineConstantPoolEntry())148 return nullptr;149 150 return CPE.Val.ConstVal;151}152 153bool StaticDataSplitter::partitionStaticDataWithProfiles(MachineFunction &MF) {154 // If any of the static data (jump tables, global variables, constant pools)155 // are captured by the analysis, set `Changed` to true. Note this pass won't156 // invalidate any analysis pass (see `getAnalysisUsage` above), so the main157 // purpose of tracking and conveying the change (to pass manager) is158 // informative as opposed to invalidating any analysis results. As an example159 // of where this information is useful, `PMDataManager::dumpPassInfo` will160 // only dump pass info if a local change happens, otherwise a pass appears as161 // "skipped".162 bool Changed = false;163 164 MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();165 166 // Jump table could be used by either terminating instructions or167 // non-terminating ones, so we walk all instructions and use168 // `MachineOperand::isJTI()` to identify jump table operands.169 // Similarly, `MachineOperand::isCPI()` is used to identify constant pool170 // usages in the same loop.171 for (const auto &MBB : MF) {172 std::optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB);173 for (const MachineInstr &I : MBB) {174 for (const MachineOperand &Op : I.operands()) {175 if (!Op.isJTI() && !Op.isGlobal() && !Op.isCPI())176 continue;177 178 if (Op.isJTI()) {179 assert(MJTI != nullptr && "Jump table info is not available.");180 const int JTI = Op.getIndex();181 // This is not a source block of jump table.182 if (JTI == -1)183 continue;184 185 auto Hotness = MachineFunctionDataHotness::Hot;186 187 // Hotness is based on source basic block hotness.188 // TODO: PSI APIs are about instruction hotness. Introduce API for189 // data access hotness.190 if (Count && PSI->isColdCount(*Count))191 Hotness = MachineFunctionDataHotness::Cold;192 193 Changed |= MJTI->updateJumpTableEntryHotness(JTI, Hotness);194 } else if (const Constant *C =195 getConstant(Op, MF.getTarget(), MF.getConstantPool())) {196 SDPI->addConstantProfileCount(C, Count);197 Changed = true;198 }199 }200 }201 }202 return Changed;203}204 205const GlobalVariable *206StaticDataSplitter::getLocalLinkageGlobalVariable(const GlobalValue *GV) {207 // LLVM IR Verifier requires that a declaration must have valid declaration208 // linkage, and local linkages are not among the valid ones. So there is no209 // need to check GV is not a declaration here.210 return (GV && GV->hasLocalLinkage()) ? dyn_cast<GlobalVariable>(GV) : nullptr;211}212 213bool StaticDataSplitter::inStaticDataSection(const GlobalVariable &GV,214 const TargetMachine &TM) {215 216 SectionKind Kind = TargetLoweringObjectFile::getKindForGlobal(&GV, TM);217 return Kind.isData() || Kind.isReadOnly() || Kind.isReadOnlyWithRel() ||218 Kind.isBSS();219}220 221void StaticDataSplitter::updateStatsWithProfiles(const MachineFunction &MF) {222 if (!AreStatisticsEnabled())223 return;224 225 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {226 for (const auto &JumpTable : MJTI->getJumpTables()) {227 if (JumpTable.Hotness == MachineFunctionDataHotness::Hot) {228 ++NumHotJumpTables;229 } else {230 assert(JumpTable.Hotness == MachineFunctionDataHotness::Cold &&231 "A jump table is either hot or cold when profile information is "232 "available.");233 ++NumColdJumpTables;234 }235 }236 }237}238 239void StaticDataSplitter::annotateStaticDataWithoutProfiles(240 const MachineFunction &MF) {241 for (const auto &MBB : MF)242 for (const MachineInstr &I : MBB)243 for (const MachineOperand &Op : I.operands())244 if (const Constant *C =245 getConstant(Op, MF.getTarget(), MF.getConstantPool()))246 SDPI->addConstantProfileCount(C, std::nullopt);247}248 249void StaticDataSplitter::updateStatsWithoutProfiles(const MachineFunction &MF) {250 if (!AreStatisticsEnabled())251 return;252 253 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {254 NumUnknownJumpTables += MJTI->getJumpTables().size();255 }256}257 258char StaticDataSplitter::ID = 0;259 260INITIALIZE_PASS_BEGIN(StaticDataSplitter, DEBUG_TYPE, "Split static data",261 false, false)262INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass)263INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass)264INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)265INITIALIZE_PASS_DEPENDENCY(StaticDataProfileInfoWrapperPass)266INITIALIZE_PASS_END(StaticDataSplitter, DEBUG_TYPE, "Split static data", false,267 false)268 269MachineFunctionPass *llvm::createStaticDataSplitterPass() {270 return new StaticDataSplitter();271}272