230 lines · cpp
1//===-- MachineFunctionSplitter.cpp - Split machine functions //-----------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// \file10// Uses profile information to split out cold blocks.11//12// This pass splits out cold machine basic blocks from the parent function. This13// implementation leverages the basic block section framework. Blocks marked14// cold by this pass are grouped together in a separate section prefixed with15// ".text.unlikely.*". The linker can then group these together as a cold16// section. The split part of the function is a contiguous region identified by17// the symbol "foo.cold". Grouping all cold blocks across functions together18// decreases fragmentation and improves icache and itlb utilization. Note that19// the overall changes to the binary size are negligible; only a small number of20// additional jump instructions may be introduced.21//22// For the original RFC of this pass please see23// https://groups.google.com/d/msg/llvm-dev/RUegaMg-iqc/wFAVxa6fCgAJ24//===----------------------------------------------------------------------===//25 26#include "llvm/ADT/SmallVector.h"27#include "llvm/Analysis/BranchProbabilityInfo.h"28#include "llvm/Analysis/EHUtils.h"29#include "llvm/Analysis/ProfileSummaryInfo.h"30#include "llvm/CodeGen/BasicBlockSectionUtils.h"31#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"32#include "llvm/CodeGen/MachineBasicBlock.h"33#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"34#include "llvm/CodeGen/MachineFunction.h"35#include "llvm/CodeGen/MachineFunctionPass.h"36#include "llvm/CodeGen/MachineModuleInfo.h"37#include "llvm/CodeGen/Passes.h"38#include "llvm/CodeGen/TargetInstrInfo.h"39#include "llvm/IR/Function.h"40#include "llvm/InitializePasses.h"41#include "llvm/Support/CommandLine.h"42#include <optional>43 44using namespace llvm;45 46// FIXME: This cutoff value is CPU dependent and should be moved to47// TargetTransformInfo once we consider enabling this on other platforms.48// The value is expressed as a ProfileSummaryInfo integer percentile cutoff.49// Defaults to 999950, i.e. all blocks colder than 99.995 percentile are split.50// The default was empirically determined to be optimal when considering cutoff51// values between 99%-ile to 100%-ile with respect to iTLB and icache metrics on52// Intel CPUs.53static cl::opt<unsigned>54 PercentileCutoff("mfs-psi-cutoff",55 cl::desc("Percentile profile summary cutoff used to "56 "determine cold blocks. Unused if set to zero."),57 cl::init(999950), cl::Hidden);58 59static cl::opt<unsigned> ColdCountThreshold(60 "mfs-count-threshold",61 cl::desc(62 "Minimum number of times a block must be executed to be retained."),63 cl::init(1), cl::Hidden);64 65static cl::opt<bool> SplitAllEHCode(66 "mfs-split-ehcode",67 cl::desc("Splits all EH code and it's descendants by default."),68 cl::init(false), cl::Hidden);69 70namespace {71 72class MachineFunctionSplitter : public MachineFunctionPass {73public:74 static char ID;75 MachineFunctionSplitter() : MachineFunctionPass(ID) {76 initializeMachineFunctionSplitterPass(*PassRegistry::getPassRegistry());77 }78 79 StringRef getPassName() const override {80 return "Machine Function Splitter Transformation";81 }82 83 void getAnalysisUsage(AnalysisUsage &AU) const override;84 85 bool runOnMachineFunction(MachineFunction &F) override;86};87} // end anonymous namespace88 89/// setDescendantEHBlocksCold - This splits all EH pads and blocks reachable90/// only by EH pad as cold. This will help mark EH pads statically cold91/// instead of relying on profile data.92static void setDescendantEHBlocksCold(MachineFunction &MF) {93 DenseSet<MachineBasicBlock *> EHBlocks;94 computeEHOnlyBlocks(MF, EHBlocks);95 for (auto Block : EHBlocks) {96 Block->setSectionID(MBBSectionID::ColdSectionID);97 }98}99 100static void finishAdjustingBasicBlocksAndLandingPads(MachineFunction &MF) {101 auto Comparator = [](const MachineBasicBlock &X, const MachineBasicBlock &Y) {102 return X.getSectionID().Type < Y.getSectionID().Type;103 };104 llvm::sortBasicBlocksAndUpdateBranches(MF, Comparator);105 llvm::avoidZeroOffsetLandingPad(MF);106}107 108static bool isColdBlock(const MachineBasicBlock &MBB,109 const MachineBlockFrequencyInfo *MBFI,110 ProfileSummaryInfo *PSI) {111 std::optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB);112 // For instrumentation profiles and sample profiles, we use different ways113 // to judge whether a block is cold and should be split.114 if (PSI->hasInstrumentationProfile() || PSI->hasCSInstrumentationProfile()) {115 // If using instrument profile, which is deemed "accurate", no count means116 // cold.117 if (!Count)118 return true;119 if (PercentileCutoff > 0)120 return PSI->isColdCountNthPercentile(PercentileCutoff, *Count);121 // Fallthrough to end of function.122 } else if (PSI->hasSampleProfile()) {123 // For sample profile, no count means "do not judege coldness".124 if (!Count)125 return false;126 }127 128 return (*Count < ColdCountThreshold);129}130 131bool MachineFunctionSplitter::runOnMachineFunction(MachineFunction &MF) {132 if (skipFunction(MF.getFunction()))133 return false;134 135 // Do not split functions when -basic-block-sections=all is specified.136 if (MF.getTarget().getBBSectionsType() == llvm::BasicBlockSection::All)137 return false;138 // We target functions with profile data. Static information in the form139 // of exception handling code may be split to cold if user passes the140 // mfs-split-ehcode flag.141 bool UseProfileData = MF.getFunction().hasProfileData();142 if (!UseProfileData && !SplitAllEHCode)143 return false;144 145 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();146 if (!TII.isFunctionSafeToSplit(MF))147 return false;148 149 // Do not split functions with BasicBlockSections profiles as they will150 // be split by the BasicBlockSections pass.151 auto BBSectionsProfile =152 getAnalysisIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>();153 if (BBSectionsProfile != nullptr &&154 BBSectionsProfile->getBBSPR().isFunctionHot(MF.getName()))155 return false;156 157 // Renumbering blocks here preserves the order of the blocks as158 // sortBasicBlocksAndUpdateBranches uses the numeric identifier to sort159 // blocks. Preserving the order of blocks is essential to retaining decisions160 // made by prior passes such as MachineBlockPlacement.161 MF.RenumberBlocks();162 MF.setBBSectionsType(BasicBlockSection::Preset);163 164 MachineBlockFrequencyInfo *MBFI = nullptr;165 ProfileSummaryInfo *PSI = nullptr;166 if (UseProfileData) {167 MBFI = &getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI();168 PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();169 // If we don't have a good profile (sample profile is not deemed170 // as a "good profile") and the function is not hot, then early171 // return. (Because we can only trust hot functions when profile172 // quality is not good.)173 if (PSI->hasSampleProfile() && !PSI->isFunctionHotInCallGraph(&MF, *MBFI)) {174 // Split all EH code and it's descendant statically by default.175 if (SplitAllEHCode)176 setDescendantEHBlocksCold(MF);177 finishAdjustingBasicBlocksAndLandingPads(MF);178 return true;179 }180 }181 182 SmallVector<MachineBasicBlock *, 2> LandingPads;183 for (auto &MBB : MF) {184 if (MBB.isEntryBlock())185 continue;186 187 if (MBB.isEHPad())188 LandingPads.push_back(&MBB);189 else if (UseProfileData && isColdBlock(MBB, MBFI, PSI) &&190 TII.isMBBSafeToSplitToCold(MBB) && !SplitAllEHCode)191 MBB.setSectionID(MBBSectionID::ColdSectionID);192 }193 194 // Split all EH code and it's descendant statically by default.195 if (SplitAllEHCode)196 setDescendantEHBlocksCold(MF);197 // We only split out eh pads if all of them are cold.198 else {199 // Here we have UseProfileData == true.200 bool HasHotLandingPads = false;201 for (const MachineBasicBlock *LP : LandingPads) {202 if (!isColdBlock(*LP, MBFI, PSI) || !TII.isMBBSafeToSplitToCold(*LP))203 HasHotLandingPads = true;204 }205 if (!HasHotLandingPads) {206 for (MachineBasicBlock *LP : LandingPads)207 LP->setSectionID(MBBSectionID::ColdSectionID);208 }209 }210 211 finishAdjustingBasicBlocksAndLandingPads(MF);212 return true;213}214 215void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const {216 AU.addRequired<MachineModuleInfoWrapperPass>();217 AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();218 AU.addRequired<ProfileSummaryInfoWrapperPass>();219 AU.addUsedIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>();220}221 222char MachineFunctionSplitter::ID = 0;223INITIALIZE_PASS(MachineFunctionSplitter, "machine-function-splitter",224 "Split machine functions using profile information", false,225 false)226 227MachineFunctionPass *llvm::createMachineFunctionSplitterPass() {228 return new MachineFunctionSplitter();229}230