194 lines · cpp
1#include "llvm/Analysis/StaticDataProfileInfo.h"2#include "llvm/Analysis/ProfileSummaryInfo.h"3#include "llvm/IR/Constant.h"4#include "llvm/IR/Constants.h"5#include "llvm/IR/GlobalVariable.h"6#include "llvm/IR/Module.h"7#include "llvm/InitializePasses.h"8#include "llvm/ProfileData/InstrProf.h"9 10#define DEBUG_TYPE "static-data-profile-info"11 12using namespace llvm;13 14namespace llvm {15namespace memprof {16// Returns true iff the global variable has custom section either by17// __attribute__((section("name")))18// (https://clang.llvm.org/docs/AttributeReference.html#section-declspec-allocate)19// or #pragma clang section directives20// (https://clang.llvm.org/docs/LanguageExtensions.html#specifying-section-names-for-global-objects-pragma-clang-section).21static bool hasExplicitSectionName(const GlobalVariable &GVar) {22 if (GVar.hasSection())23 return true;24 25 auto Attrs = GVar.getAttributes();26 if (Attrs.hasAttribute("bss-section") || Attrs.hasAttribute("data-section") ||27 Attrs.hasAttribute("relro-section") ||28 Attrs.hasAttribute("rodata-section"))29 return true;30 return false;31}32 33AnnotationKind getAnnotationKind(const GlobalVariable &GV) {34 if (GV.isDeclarationForLinker())35 return AnnotationKind::DeclForLinker;36 // Skip 'llvm.'-prefixed global variables conservatively because they are37 // often handled specially,38 StringRef Name = GV.getName();39 if (Name.starts_with("llvm."))40 return AnnotationKind::ReservedName;41 // Respect user-specified custom data sections.42 if (hasExplicitSectionName(GV))43 return AnnotationKind::ExplicitSection;44 return AnnotationKind::AnnotationOK;45}46 47bool IsAnnotationOK(const GlobalVariable &GV) {48 return getAnnotationKind(GV) == AnnotationKind::AnnotationOK;49}50} // namespace memprof51} // namespace llvm52 53void StaticDataProfileInfo::addConstantProfileCount(54 const Constant *C, std::optional<uint64_t> Count) {55 if (!Count) {56 ConstantWithoutCounts.insert(C);57 return;58 }59 uint64_t &OriginalCount = ConstantProfileCounts[C];60 OriginalCount = llvm::SaturatingAdd(*Count, OriginalCount);61 // Clamp the count to getInstrMaxCountValue. InstrFDO reserves a few62 // large values for special use.63 if (OriginalCount > getInstrMaxCountValue())64 OriginalCount = getInstrMaxCountValue();65}66 67StaticDataProfileInfo::StaticDataHotness68StaticDataProfileInfo::getConstantHotnessUsingProfileCount(69 const Constant *C, const ProfileSummaryInfo *PSI, uint64_t Count) const {70 // The accummulated counter shows the constant is hot. Return enum 'hot'71 // whether this variable is seen by unprofiled functions or not.72 if (PSI->isHotCount(Count))73 return StaticDataHotness::Hot;74 // The constant is not hot, and seen by unprofiled functions. We don't want to75 // assign it to unlikely sections, even if the counter says 'cold'. So return76 // enum 'LukewarmOrUnknown'.77 if (ConstantWithoutCounts.count(C))78 return StaticDataHotness::LukewarmOrUnknown;79 // The accummulated counter shows the constant is cold so return enum 'cold'.80 if (PSI->isColdCount(Count))81 return StaticDataHotness::Cold;82 83 return StaticDataHotness::LukewarmOrUnknown;84}85 86StaticDataProfileInfo::StaticDataHotness87StaticDataProfileInfo::getSectionHotnessUsingDataAccessProfile(88 std::optional<StringRef> MaybeSectionPrefix) const {89 if (!MaybeSectionPrefix)90 return StaticDataHotness::LukewarmOrUnknown;91 StringRef Prefix = *MaybeSectionPrefix;92 assert((Prefix == "hot" || Prefix == "unlikely") &&93 "Expect section_prefix to be one of hot or unlikely");94 return Prefix == "hot" ? StaticDataHotness::Hot : StaticDataHotness::Cold;95}96 97StringRef StaticDataProfileInfo::hotnessToStr(StaticDataHotness Hotness) const {98 switch (Hotness) {99 case StaticDataHotness::Cold:100 return "unlikely";101 case StaticDataHotness::Hot:102 return "hot";103 default:104 return "";105 }106}107 108std::optional<uint64_t>109StaticDataProfileInfo::getConstantProfileCount(const Constant *C) const {110 auto I = ConstantProfileCounts.find(C);111 if (I == ConstantProfileCounts.end())112 return std::nullopt;113 return I->second;114}115 116StringRef StaticDataProfileInfo::getConstantSectionPrefix(117 const Constant *C, const ProfileSummaryInfo *PSI) const {118 std::optional<uint64_t> Count = getConstantProfileCount(C);119 120#ifndef NDEBUG121 auto DbgPrintPrefix = [](StringRef Prefix) {122 return Prefix.empty() ? "<empty>" : Prefix;123 };124#endif125 126 if (EnableDataAccessProf) {127 // Module flag `HasDataAccessProf` is 1 -> empty section prefix means128 // unknown hotness except for string literals.129 if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C);130 GV && llvm::memprof::IsAnnotationOK(*GV) &&131 !GV->getName().starts_with(".str")) {132 auto HotnessFromDataAccessProf =133 getSectionHotnessUsingDataAccessProfile(GV->getSectionPrefix());134 135 if (!Count) {136 StringRef Prefix = hotnessToStr(HotnessFromDataAccessProf);137 LLVM_DEBUG(dbgs() << GV->getName() << " has section prefix "138 << DbgPrintPrefix(Prefix)139 << ", solely from data access profiles\n");140 return Prefix;141 }142 143 // Both data access profiles and PGO counters are available. Use the144 // hotter one.145 auto HotnessFromPGO = getConstantHotnessUsingProfileCount(C, PSI, *Count);146 StaticDataHotness GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;147 if (HotnessFromDataAccessProf == StaticDataHotness::Hot ||148 HotnessFromPGO == StaticDataHotness::Hot) {149 GlobalVarHotness = StaticDataHotness::Hot;150 } else if (HotnessFromDataAccessProf ==151 StaticDataHotness::LukewarmOrUnknown ||152 HotnessFromPGO == StaticDataHotness::LukewarmOrUnknown) {153 GlobalVarHotness = StaticDataHotness::LukewarmOrUnknown;154 } else {155 GlobalVarHotness = StaticDataHotness::Cold;156 }157 StringRef Prefix = hotnessToStr(GlobalVarHotness);158 LLVM_DEBUG(159 dbgs() << GV->getName() << " has section prefix "160 << DbgPrintPrefix(Prefix)161 << ", the max from data access profiles as "162 << DbgPrintPrefix(hotnessToStr(HotnessFromDataAccessProf))163 << " and PGO counters as "164 << DbgPrintPrefix(hotnessToStr(HotnessFromPGO)) << "\n");165 return Prefix;166 }167 }168 if (!Count)169 return "";170 return hotnessToStr(getConstantHotnessUsingProfileCount(C, PSI, *Count));171}172 173bool StaticDataProfileInfoWrapperPass::doInitialization(Module &M) {174 bool EnableDataAccessProf = false;175 if (auto *MD = mdconst::extract_or_null<ConstantInt>(176 M.getModuleFlag("EnableDataAccessProf")))177 EnableDataAccessProf = MD->getZExtValue();178 Info.reset(new StaticDataProfileInfo(EnableDataAccessProf));179 return false;180}181 182bool StaticDataProfileInfoWrapperPass::doFinalization(Module &M) {183 Info.reset();184 return false;185}186 187INITIALIZE_PASS(StaticDataProfileInfoWrapperPass, "static-data-profile-info",188 "Static Data Profile Info", false, true)189 190StaticDataProfileInfoWrapperPass::StaticDataProfileInfoWrapperPass()191 : ImmutablePass(ID) {}192 193char StaticDataProfileInfoWrapperPass::ID = 0;194