brintos / llvm-project-archived public Read only

1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This pass lowers instrprof_* intrinsics emitted by an instrumentor.10// It also builds the data structures and initialization code needed for11// updating execution counts and emitting the profile at runtime.12//13//===----------------------------------------------------------------------===//14 15#include "llvm/Transforms/Instrumentation/InstrProfiling.h"16#include "llvm/ADT/ArrayRef.h"17#include "llvm/ADT/STLExtras.h"18#include "llvm/ADT/SmallVector.h"19#include "llvm/ADT/StringRef.h"20#include "llvm/ADT/Twine.h"21#include "llvm/Analysis/BlockFrequencyInfo.h"22#include "llvm/Analysis/BranchProbabilityInfo.h"23#include "llvm/Analysis/CFG.h"24#include "llvm/Analysis/LoopInfo.h"25#include "llvm/Analysis/TargetLibraryInfo.h"26#include "llvm/IR/Attributes.h"27#include "llvm/IR/BasicBlock.h"28#include "llvm/IR/CFG.h"29#include "llvm/IR/Constant.h"30#include "llvm/IR/Constants.h"31#include "llvm/IR/DIBuilder.h"32#include "llvm/IR/DerivedTypes.h"33#include "llvm/IR/DiagnosticInfo.h"34#include "llvm/IR/Dominators.h"35#include "llvm/IR/Function.h"36#include "llvm/IR/GlobalValue.h"37#include "llvm/IR/GlobalVariable.h"38#include "llvm/IR/IRBuilder.h"39#include "llvm/IR/Instruction.h"40#include "llvm/IR/Instructions.h"41#include "llvm/IR/IntrinsicInst.h"42#include "llvm/IR/MDBuilder.h"43#include "llvm/IR/Module.h"44#include "llvm/IR/Type.h"45#include "llvm/Pass.h"46#include "llvm/ProfileData/InstrProf.h"47#include "llvm/ProfileData/InstrProfCorrelator.h"48#include "llvm/Support/Casting.h"49#include "llvm/Support/CommandLine.h"50#include "llvm/Support/Compiler.h"51#include "llvm/Support/Error.h"52#include "llvm/Support/ErrorHandling.h"53#include "llvm/TargetParser/Triple.h"54#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"55#include "llvm/Transforms/Utils/BasicBlockUtils.h"56#include "llvm/Transforms/Utils/Instrumentation.h"57#include "llvm/Transforms/Utils/ModuleUtils.h"58#include "llvm/Transforms/Utils/SSAUpdater.h"59#include <algorithm>60#include <cassert>61#include <cstdint>62#include <string>63 64using namespace llvm;65 66#define DEBUG_TYPE "instrprof"67 68namespace llvm {69// Command line option to enable vtable value profiling. Defined in70// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=71extern cl::opt<bool> EnableVTableValueProfiling;72LLVM_ABI cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate(73    "profile-correlate",74    cl::desc("Use debug info or binary file to correlate profiles."),75    cl::init(InstrProfCorrelator::NONE),76    cl::values(clEnumValN(InstrProfCorrelator::NONE, "",77                          "No profile correlation"),78               clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info",79                          "Use debug info to correlate"),80               clEnumValN(InstrProfCorrelator::BINARY, "binary",81                          "Use binary to correlate")));82} // namespace llvm83 84namespace {85 86cl::opt<bool> DoHashBasedCounterSplit(87    "hash-based-counter-split",88    cl::desc("Rename counter variable of a comdat function based on cfg hash"),89    cl::init(true));90 91cl::opt<bool>92    RuntimeCounterRelocation("runtime-counter-relocation",93                             cl::desc("Enable relocating counters at runtime."),94                             cl::init(false));95 96cl::opt<bool> ValueProfileStaticAlloc(97    "vp-static-alloc",98    cl::desc("Do static counter allocation for value profiler"),99    cl::init(true));100 101cl::opt<double> NumCountersPerValueSite(102    "vp-counters-per-site",103    cl::desc("The average number of profile counters allocated "104             "per value profiling site."),105    // This is set to a very small value because in real programs, only106    // a very small percentage of value sites have non-zero targets, e.g, 1/30.107    // For those sites with non-zero profile, the average number of targets108    // is usually smaller than 2.109    cl::init(1.0));110 111cl::opt<bool> AtomicCounterUpdateAll(112    "instrprof-atomic-counter-update-all",113    cl::desc("Make all profile counter updates atomic (for testing only)"),114    cl::init(false));115 116cl::opt<bool> AtomicCounterUpdatePromoted(117    "atomic-counter-update-promoted",118    cl::desc("Do counter update using atomic fetch add "119             " for promoted counters only"),120    cl::init(false));121 122cl::opt<bool> AtomicFirstCounter(123    "atomic-first-counter",124    cl::desc("Use atomic fetch add for first counter in a function (usually "125             "the entry counter)"),126    cl::init(false));127 128cl::opt<bool> ConditionalCounterUpdate(129    "conditional-counter-update",130    cl::desc("Do conditional counter updates in single byte counters mode)"),131    cl::init(false));132 133// If the option is not specified, the default behavior about whether134// counter promotion is done depends on how instrumentation lowering135// pipeline is setup, i.e., the default value of true of this option136// does not mean the promotion will be done by default. Explicitly137// setting this option can override the default behavior.138cl::opt<bool> DoCounterPromotion("do-counter-promotion",139                                 cl::desc("Do counter register promotion"),140                                 cl::init(false));141cl::opt<unsigned> MaxNumOfPromotionsPerLoop(142    "max-counter-promotions-per-loop", cl::init(20),143    cl::desc("Max number counter promotions per loop to avoid"144             " increasing register pressure too much"));145 146// A debug option147cl::opt<int>148    MaxNumOfPromotions("max-counter-promotions", cl::init(-1),149                       cl::desc("Max number of allowed counter promotions"));150 151cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(152    "speculative-counter-promotion-max-exiting", cl::init(3),153    cl::desc("The max number of exiting blocks of a loop to allow "154             " speculative counter promotion"));155 156cl::opt<bool> SpeculativeCounterPromotionToLoop(157    "speculative-counter-promotion-to-loop",158    cl::desc("When the option is false, if the target block is in a loop, "159             "the promotion will be disallowed unless the promoted counter "160             " update can be further/iteratively promoted into an acyclic "161             " region."));162 163cl::opt<bool> IterativeCounterPromotion(164    "iterative-counter-promotion", cl::init(true),165    cl::desc("Allow counter promotion across the whole loop nest."));166 167cl::opt<bool> SkipRetExitBlock(168    "skip-ret-exit-block", cl::init(true),169    cl::desc("Suppress counter promotion if exit blocks contain ret."));170 171static cl::opt<bool> SampledInstr("sampled-instrumentation", cl::ZeroOrMore,172                                  cl::init(false),173                                  cl::desc("Do PGO instrumentation sampling"));174 175static cl::opt<unsigned> SampledInstrPeriod(176    "sampled-instr-period",177    cl::desc("Set the profile instrumentation sample period. A sample period "178             "of 0 is invalid. For each sample period, a fixed number of "179             "consecutive samples will be recorded. The number is controlled "180             "by 'sampled-instr-burst-duration' flag. The default sample "181             "period of 65536 is optimized for generating efficient code that "182             "leverages unsigned short integer wrapping in overflow, but this "183             "is disabled under simple sampling (burst duration = 1)."),184    cl::init(USHRT_MAX + 1));185 186static cl::opt<unsigned> SampledInstrBurstDuration(187    "sampled-instr-burst-duration",188    cl::desc("Set the profile instrumentation burst duration, which can range "189             "from 1 to the value of 'sampled-instr-period' (0 is invalid). "190             "This number of samples will be recorded for each "191             "'sampled-instr-period' count update. Setting to 1 enables simple "192             "sampling, in which case it is recommended to set "193             "'sampled-instr-period' to a prime number."),194    cl::init(200));195 196struct SampledInstrumentationConfig {197  unsigned BurstDuration;198  unsigned Period;199  bool UseShort;200  bool IsSimpleSampling;201  bool IsFastSampling;202};203 204static SampledInstrumentationConfig getSampledInstrumentationConfig() {205  SampledInstrumentationConfig config;206  config.BurstDuration = SampledInstrBurstDuration.getValue();207  config.Period = SampledInstrPeriod.getValue();208  if (config.BurstDuration > config.Period)209    report_fatal_error(210        "SampledBurstDuration must be less than or equal to SampledPeriod");211  if (config.Period == 0 || config.BurstDuration == 0)212    report_fatal_error(213        "SampledPeriod and SampledBurstDuration must be greater than 0");214  config.IsSimpleSampling = (config.BurstDuration == 1);215  // If (BurstDuration == 1 && Period == 65536), generate the simple sampling216  // style code.217  config.IsFastSampling =218      (!config.IsSimpleSampling && config.Period == USHRT_MAX + 1);219  config.UseShort = (config.Period <= USHRT_MAX) || config.IsFastSampling;220  return config;221}222 223using LoadStorePair = std::pair<Instruction *, Instruction *>;224 225static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {226  auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));227  if (!MD)228    return 0;229 230  // If the flag is a ConstantAsMetadata, it should be an integer representable231  // in 64-bits.232  return cast<ConstantInt>(MD->getValue())->getZExtValue();233}234 235static bool enablesValueProfiling(const Module &M) {236  return isIRPGOFlagSet(&M) ||237         getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;238}239 240// Conservatively returns true if value profiling is enabled.241static bool profDataReferencedByCode(const Module &M) {242  return enablesValueProfiling(M);243}244 245class InstrLowerer final {246public:247  InstrLowerer(Module &M, const InstrProfOptions &Options,248               std::function<const TargetLibraryInfo &(Function &F)> GetTLI,249               bool IsCS)250      : M(M), Options(Options), TT(M.getTargetTriple()), IsCS(IsCS),251        GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {}252 253  bool lower();254 255private:256  Module &M;257  const InstrProfOptions Options;258  const Triple TT;259  // Is this lowering for the context-sensitive instrumentation.260  const bool IsCS;261 262  std::function<const TargetLibraryInfo &(Function &F)> GetTLI;263 264  const bool DataReferencedByCode;265 266  struct PerFunctionProfileData {267    uint32_t NumValueSites[IPVK_Last + 1] = {};268    GlobalVariable *RegionCounters = nullptr;269    GlobalVariable *DataVar = nullptr;270    GlobalVariable *RegionBitmaps = nullptr;271    uint32_t NumBitmapBytes = 0;272 273    PerFunctionProfileData() = default;274  };275  DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;276  // Key is virtual table variable, value is 'VTableProfData' in the form of277  // GlobalVariable.278  DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap;279  /// If runtime relocation is enabled, this maps functions to the load280  /// instruction that produces the profile relocation bias.281  DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;282  std::vector<GlobalValue *> CompilerUsedVars;283  std::vector<GlobalValue *> UsedVars;284  std::vector<GlobalVariable *> ReferencedNames;285  // The list of virtual table variables of which the VTableProfData is286  // collected.287  std::vector<GlobalVariable *> ReferencedVTables;288  GlobalVariable *NamesVar = nullptr;289  size_t NamesSize = 0;290 291  // vector of counter load/store pairs to be register promoted.292  std::vector<LoadStorePair> PromotionCandidates;293 294  int64_t TotalCountersPromoted = 0;295 296  /// Lower instrumentation intrinsics in the function. Returns true if there297  /// any lowering.298  bool lowerIntrinsics(Function *F);299 300  /// Register-promote counter loads and stores in loops.301  void promoteCounterLoadStores(Function *F);302 303  /// Returns true if relocating counters at runtime is enabled.304  bool isRuntimeCounterRelocationEnabled() const;305 306  /// Returns true if profile counter update register promotion is enabled.307  bool isCounterPromotionEnabled() const;308 309  /// Return true if profile sampling is enabled.310  bool isSamplingEnabled() const;311 312  /// Count the number of instrumented value sites for the function.313  void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);314 315  /// Replace instrprof.value.profile with a call to runtime library.316  void lowerValueProfileInst(InstrProfValueProfileInst *Ins);317 318  /// Replace instrprof.cover with a store instruction to the coverage byte.319  void lowerCover(InstrProfCoverInst *Inc);320 321  /// Replace instrprof.timestamp with a call to322  /// INSTR_PROF_PROFILE_SET_TIMESTAMP.323  void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction);324 325  /// Replace instrprof.increment with an increment of the appropriate value.326  void lowerIncrement(InstrProfIncrementInst *Inc);327 328  /// Force emitting of name vars for unused functions.329  void lowerCoverageData(GlobalVariable *CoverageNamesVar);330 331  /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction332  /// using the index represented by the a temp value into a bitmap.333  void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins);334 335  /// Get the Bias value for data to access mmap-ed area.336  /// Create it if it hasn't been seen.337  GlobalVariable *getOrCreateBiasVar(StringRef VarName);338 339  /// Compute the address of the counter value that this profiling instruction340  /// acts on.341  Value *getCounterAddress(InstrProfCntrInstBase *I);342 343  /// Lower the incremental instructions under profile sampling predicates.344  void doSampling(Instruction *I);345 346  /// Get the region counters for an increment, creating them if necessary.347  ///348  /// If the counter array doesn't yet exist, the profile data variables349  /// referring to them will also be created.350  GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc);351 352  /// Create the region counters.353  GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc,354                                       StringRef Name,355                                       GlobalValue::LinkageTypes Linkage);356 357  /// Compute the address of the test vector bitmap that this profiling358  /// instruction acts on.359  Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);360 361  /// Get the region bitmaps for an increment, creating them if necessary.362  ///363  /// If the bitmap array doesn't yet exist, the profile data variables364  /// referring to them will also be created.365  GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc);366 367  /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with368  /// an MC/DC Decision region. The number of bytes required is indicated by369  /// the intrinsic used (type InstrProfMCDCBitmapInstBase).  This is called370  /// as part of setupProfileSection() and is conceptually very similar to371  /// what is done for profile data counters in createRegionCounters().372  GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,373                                      StringRef Name,374                                      GlobalValue::LinkageTypes Linkage);375 376  /// Set Comdat property of GV, if required.377  void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName);378 379  /// Setup the sections into which counters and bitmaps are allocated.380  GlobalVariable *setupProfileSection(InstrProfInstBase *Inc,381                                      InstrProfSectKind IPSK);382 383  /// Create INSTR_PROF_DATA variable for counters and bitmaps.384  void createDataVariable(InstrProfCntrInstBase *Inc);385 386  /// Get the counters for virtual table values, creating them if necessary.387  void getOrCreateVTableProfData(GlobalVariable *GV);388 389  /// Emit the section with compressed function names.390  void emitNameData();391 392  /// Emit the section with compressed vtable names.393  void emitVTableNames();394 395  /// Emit value nodes section for value profiling.396  void emitVNodes();397 398  /// Emit runtime registration functions for each profile data variable.399  void emitRegistration();400 401  /// Emit the necessary plumbing to pull in the runtime initialization.402  /// Returns true if a change was made.403  bool emitRuntimeHook();404 405  /// Add uses of our data variables and runtime hook.406  void emitUses();407 408  /// Create a static initializer for our data, on platforms that need it,409  /// and for any profile output file that was specified.410  void emitInitialization();411};412 413///414/// A helper class to promote one counter RMW operation in the loop415/// into register update.416///417/// RWM update for the counter will be sinked out of the loop after418/// the transformation.419///420class PGOCounterPromoterHelper : public LoadAndStorePromoter {421public:422  PGOCounterPromoterHelper(423      Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,424      BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,425      ArrayRef<Instruction *> InsertPts,426      DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,427      LoopInfo &LI)428      : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),429        InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {430    assert(isa<LoadInst>(L));431    assert(isa<StoreInst>(S));432    SSA.AddAvailableValue(PH, Init);433  }434 435  void doExtraRewritesBeforeFinalDeletion() override {436    for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {437      BasicBlock *ExitBlock = ExitBlocks[i];438      Instruction *InsertPos = InsertPts[i];439      // Get LiveIn value into the ExitBlock. If there are multiple440      // predecessors, the value is defined by a PHI node in this441      // block.442      Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);443      Value *Addr = cast<StoreInst>(Store)->getPointerOperand();444      Type *Ty = LiveInValue->getType();445      IRBuilder<> Builder(InsertPos);446      if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Addr)) {447        // If isRuntimeCounterRelocationEnabled() is true then the address of448        // the store instruction is computed with two instructions in449        // InstrProfiling::getCounterAddress(). We need to copy those450        // instructions to this block to compute Addr correctly.451        // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>452        // %Addr = inttoptr i64 %BiasAdd to i64*453        auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));454        assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);455        Value *BiasInst = Builder.Insert(OrigBiasInst->clone());456        Addr = Builder.CreateIntToPtr(BiasInst,457                                      PointerType::getUnqual(Ty->getContext()));458      }459      if (AtomicCounterUpdatePromoted)460        // automic update currently can only be promoted across the current461        // loop, not the whole loop nest.462        Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,463                                MaybeAlign(),464                                AtomicOrdering::SequentiallyConsistent);465      else {466        LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");467        auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);468        auto *NewStore = Builder.CreateStore(NewVal, Addr);469 470        // Now update the parent loop's candidate list:471        if (IterativeCounterPromotion) {472          auto *TargetLoop = LI.getLoopFor(ExitBlock);473          if (TargetLoop)474            LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);475        }476      }477    }478  }479 480private:481  Instruction *Store;482  ArrayRef<BasicBlock *> ExitBlocks;483  ArrayRef<Instruction *> InsertPts;484  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;485  LoopInfo &LI;486};487 488/// A helper class to do register promotion for all profile counter489/// updates in a loop.490///491class PGOCounterPromoter {492public:493  PGOCounterPromoter(494      DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,495      Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)496      : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {497 498    // Skip collection of ExitBlocks and InsertPts for loops that will not be499    // able to have counters promoted.500    SmallVector<BasicBlock *, 8> LoopExitBlocks;501    SmallPtrSet<BasicBlock *, 8> BlockSet;502 503    L.getExitBlocks(LoopExitBlocks);504    if (!isPromotionPossible(&L, LoopExitBlocks))505      return;506 507    for (BasicBlock *ExitBlock : LoopExitBlocks) {508      if (BlockSet.insert(ExitBlock).second &&509          llvm::none_of(predecessors(ExitBlock), [&](const BasicBlock *Pred) {510            return llvm::isPresplitCoroSuspendExitEdge(*Pred, *ExitBlock);511          })) {512        ExitBlocks.push_back(ExitBlock);513        InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());514      }515    }516  }517 518  bool run(int64_t *NumPromoted) {519    // Skip 'infinite' loops:520    if (ExitBlocks.size() == 0)521      return false;522 523    // Skip if any of the ExitBlocks contains a ret instruction.524    // This is to prevent dumping of incomplete profile -- if the525    // the loop is a long running loop and dump is called in the middle526    // of the loop, the result profile is incomplete.527    // FIXME: add other heuristics to detect long running loops.528    if (SkipRetExitBlock) {529      for (auto *BB : ExitBlocks)530        if (isa<ReturnInst>(BB->getTerminator()))531          return false;532    }533 534    unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);535    if (MaxProm == 0)536      return false;537 538    unsigned Promoted = 0;539    for (auto &Cand : LoopToCandidates[&L]) {540 541      SmallVector<PHINode *, 4> NewPHIs;542      SSAUpdater SSA(&NewPHIs);543      Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);544 545      // If BFI is set, we will use it to guide the promotions.546      if (BFI) {547        auto *BB = Cand.first->getParent();548        auto InstrCount = BFI->getBlockProfileCount(BB);549        if (!InstrCount)550          continue;551        auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());552        // If the average loop trip count is not greater than 1.5, we skip553        // promotion.554        if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))555          continue;556      }557 558      PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,559                                        L.getLoopPreheader(), ExitBlocks,560                                        InsertPts, LoopToCandidates, LI);561      Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));562      Promoted++;563      if (Promoted >= MaxProm)564        break;565 566      (*NumPromoted)++;567      if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)568        break;569    }570 571    LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="572                      << L.getLoopDepth() << ")\n");573    return Promoted != 0;574  }575 576private:577  bool allowSpeculativeCounterPromotion(Loop *LP) {578    SmallVector<BasicBlock *, 8> ExitingBlocks;579    L.getExitingBlocks(ExitingBlocks);580    // Not considierered speculative.581    if (ExitingBlocks.size() == 1)582      return true;583    if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)584      return false;585    return true;586  }587 588  // Check whether the loop satisfies the basic conditions needed to perform589  // Counter Promotions.590  bool591  isPromotionPossible(Loop *LP,592                      const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {593    // We can't insert into a catchswitch.594    if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {595          return isa<CatchSwitchInst>(Exit->getTerminator());596        }))597      return false;598 599    if (!LP->hasDedicatedExits())600      return false;601 602    BasicBlock *PH = LP->getLoopPreheader();603    if (!PH)604      return false;605 606    return true;607  }608 609  // Returns the max number of Counter Promotions for LP.610  unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {611    SmallVector<BasicBlock *, 8> LoopExitBlocks;612    LP->getExitBlocks(LoopExitBlocks);613    if (!isPromotionPossible(LP, LoopExitBlocks))614      return 0;615 616    SmallVector<BasicBlock *, 8> ExitingBlocks;617    LP->getExitingBlocks(ExitingBlocks);618 619    // If BFI is set, we do more aggressive promotions based on BFI.620    if (BFI)621      return (unsigned)-1;622 623    // Not considierered speculative.624    if (ExitingBlocks.size() == 1)625      return MaxNumOfPromotionsPerLoop;626 627    if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)628      return 0;629 630    // Whether the target block is in a loop does not matter:631    if (SpeculativeCounterPromotionToLoop)632      return MaxNumOfPromotionsPerLoop;633 634    // Now check the target block:635    unsigned MaxProm = MaxNumOfPromotionsPerLoop;636    for (auto *TargetBlock : LoopExitBlocks) {637      auto *TargetLoop = LI.getLoopFor(TargetBlock);638      if (!TargetLoop)639        continue;640      unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);641      unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();642      MaxProm =643          std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -644                                PendingCandsInTarget);645    }646    return MaxProm;647  }648 649  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;650  SmallVector<BasicBlock *, 8> ExitBlocks;651  SmallVector<Instruction *, 8> InsertPts;652  Loop &L;653  LoopInfo &LI;654  BlockFrequencyInfo *BFI;655};656 657enum class ValueProfilingCallType {658  // Individual values are tracked. Currently used for indiret call target659  // profiling.660  Default,661 662  // MemOp: the memop size value profiling.663  MemOp664};665 666} // end anonymous namespace667 668PreservedAnalyses InstrProfilingLoweringPass::run(Module &M,669                                                  ModuleAnalysisManager &AM) {670  FunctionAnalysisManager &FAM =671      AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();672  auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {673    return FAM.getResult<TargetLibraryAnalysis>(F);674  };675  InstrLowerer Lowerer(M, Options, GetTLI, IsCS);676  if (!Lowerer.lower())677    return PreservedAnalyses::all();678 679  return PreservedAnalyses::none();680}681 682//683// Perform instrumentation sampling.684//685// There are 3 favors of sampling:686// (1) Full burst sampling: We transform:687//   Increment_Instruction;688// to:689//   if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {690//     Increment_Instruction;691//   }692//   __llvm_profile_sampling__ += 1;693//   if (__llvm_profile_sampling__ >= SampledInstrPeriod) {694//     __llvm_profile_sampling__ = 0;695//   }696//697// "__llvm_profile_sampling__" is a thread-local global shared by all PGO698// counters (value-instrumentation and edge instrumentation).699//700// (2) Fast burst sampling:701// "__llvm_profile_sampling__" variable is an unsigned type, meaning it will702// wrap around to zero when overflows. In this case, the second check is703// unnecessary, so we won't generate check2 when the SampledInstrPeriod is704// set to 65536 (64K). The code after:705//   if (__llvm_profile_sampling__ <= SampledInstrBurstDuration - 1) {706//     Increment_Instruction;707//   }708//   __llvm_profile_sampling__ += 1;709//710// (3) Simple sampling:711// When SampledInstrBurstDuration is set to 1, we do a simple sampling:712//   __llvm_profile_sampling__ += 1;713//   if (__llvm_profile_sampling__ >= SampledInstrPeriod) {714//     __llvm_profile_sampling__ = 0;715//     Increment_Instruction;716//   }717//718// Note that, the code snippet after the transformation can still be counter719// promoted. However, with sampling enabled, counter updates are expected to720// be infrequent, making the benefits of counter promotion negligible.721// Moreover, counter promotion can potentially cause issues in server722// applications, particularly when the counters are dumped without a clean723// exit. To mitigate this risk, counter promotion is disabled by default when724// sampling is enabled. This behavior can be overridden using the internal725// option.726void InstrLowerer::doSampling(Instruction *I) {727  if (!isSamplingEnabled())728    return;729 730  SampledInstrumentationConfig config = getSampledInstrumentationConfig();731  auto GetConstant = [&config](IRBuilder<> &Builder, uint32_t C) {732    if (config.UseShort)733      return Builder.getInt16(C);734    else735      return Builder.getInt32(C);736  };737 738  IntegerType *SamplingVarTy;739  if (config.UseShort)740    SamplingVarTy = Type::getInt16Ty(M.getContext());741  else742    SamplingVarTy = Type::getInt32Ty(M.getContext());743  auto *SamplingVar =744      M.getGlobalVariable(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));745  assert(SamplingVar && "SamplingVar not set properly");746 747  // Create the condition for checking the burst duration.748  Instruction *SamplingVarIncr;749  Value *NewSamplingVarVal;750  MDBuilder MDB(I->getContext());751  MDNode *BranchWeight;752  IRBuilder<> CondBuilder(I);753  auto *LoadSamplingVar = CondBuilder.CreateLoad(SamplingVarTy, SamplingVar);754  if (config.IsSimpleSampling) {755    // For the simple sampling, just create the load and increments.756    IRBuilder<> IncBuilder(I);757    NewSamplingVarVal =758        IncBuilder.CreateAdd(LoadSamplingVar, GetConstant(IncBuilder, 1));759    SamplingVarIncr = IncBuilder.CreateStore(NewSamplingVarVal, SamplingVar);760  } else {761    // For the burst-sampling, create the conditional update.762    auto *DurationCond = CondBuilder.CreateICmpULE(763        LoadSamplingVar, GetConstant(CondBuilder, config.BurstDuration - 1));764    BranchWeight = MDB.createBranchWeights(765        config.BurstDuration, config.Period - config.BurstDuration);766    Instruction *ThenTerm = SplitBlockAndInsertIfThen(767        DurationCond, I, /* Unreachable */ false, BranchWeight);768    IRBuilder<> IncBuilder(I);769    NewSamplingVarVal =770        IncBuilder.CreateAdd(LoadSamplingVar, GetConstant(IncBuilder, 1));771    SamplingVarIncr = IncBuilder.CreateStore(NewSamplingVarVal, SamplingVar);772    I->moveBefore(ThenTerm->getIterator());773  }774 775  if (config.IsFastSampling)776    return;777 778  // Create the condition for checking the period.779  Instruction *ThenTerm, *ElseTerm;780  IRBuilder<> PeriodCondBuilder(SamplingVarIncr);781  auto *PeriodCond = PeriodCondBuilder.CreateICmpUGE(782      NewSamplingVarVal, GetConstant(PeriodCondBuilder, config.Period));783  BranchWeight = MDB.createBranchWeights(1, config.Period - 1);784  SplitBlockAndInsertIfThenElse(PeriodCond, SamplingVarIncr, &ThenTerm,785                                &ElseTerm, BranchWeight);786 787  // For the simple sampling, the counter update happens in sampling var reset.788  if (config.IsSimpleSampling)789    I->moveBefore(ThenTerm->getIterator());790 791  IRBuilder<> ResetBuilder(ThenTerm);792  ResetBuilder.CreateStore(GetConstant(ResetBuilder, 0), SamplingVar);793  SamplingVarIncr->moveBefore(ElseTerm->getIterator());794}795 796bool InstrLowerer::lowerIntrinsics(Function *F) {797  bool MadeChange = false;798  PromotionCandidates.clear();799  SmallVector<InstrProfInstBase *, 8> InstrProfInsts;800 801  // To ensure compatibility with sampling, we save the intrinsics into802  // a buffer to prevent potential breakage of the iterator (as the803  // intrinsics will be moved to a different BB).804  for (BasicBlock &BB : *F) {805    for (Instruction &Instr : llvm::make_early_inc_range(BB)) {806      if (auto *IP = dyn_cast<InstrProfInstBase>(&Instr))807        InstrProfInsts.push_back(IP);808    }809  }810 811  for (auto *Instr : InstrProfInsts) {812    doSampling(Instr);813    if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(Instr)) {814      lowerIncrement(IPIS);815      MadeChange = true;816    } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(Instr)) {817      lowerIncrement(IPI);818      MadeChange = true;819    } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(Instr)) {820      lowerTimestamp(IPC);821      MadeChange = true;822    } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(Instr)) {823      lowerCover(IPC);824      MadeChange = true;825    } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(Instr)) {826      lowerValueProfileInst(IPVP);827      MadeChange = true;828    } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(Instr)) {829      IPMP->eraseFromParent();830      MadeChange = true;831    } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(Instr)) {832      lowerMCDCTestVectorBitmapUpdate(IPBU);833      MadeChange = true;834    }835  }836 837  if (!MadeChange)838    return false;839 840  promoteCounterLoadStores(F);841  return true;842}843 844bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {845  // Mach-O don't support weak external references.846  if (TT.isOSBinFormatMachO())847    return false;848 849  if (RuntimeCounterRelocation.getNumOccurrences() > 0)850    return RuntimeCounterRelocation;851 852  // Fuchsia uses runtime counter relocation by default.853  return TT.isOSFuchsia();854}855 856bool InstrLowerer::isSamplingEnabled() const {857  if (SampledInstr.getNumOccurrences() > 0)858    return SampledInstr;859  return Options.Sampling;860}861 862bool InstrLowerer::isCounterPromotionEnabled() const {863  if (DoCounterPromotion.getNumOccurrences() > 0)864    return DoCounterPromotion;865 866  return Options.DoCounterPromotion;867}868 869void InstrLowerer::promoteCounterLoadStores(Function *F) {870  if (!isCounterPromotionEnabled())871    return;872 873  DominatorTree DT(*F);874  LoopInfo LI(DT);875  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;876 877  std::unique_ptr<BlockFrequencyInfo> BFI;878  if (Options.UseBFIInPromotion) {879    std::unique_ptr<BranchProbabilityInfo> BPI;880    BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));881    BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));882  }883 884  for (const auto &LoadStore : PromotionCandidates) {885    auto *CounterLoad = LoadStore.first;886    auto *CounterStore = LoadStore.second;887    BasicBlock *BB = CounterLoad->getParent();888    Loop *ParentLoop = LI.getLoopFor(BB);889    if (!ParentLoop)890      continue;891    LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);892  }893 894  SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();895 896  // Do a post-order traversal of the loops so that counter updates can be897  // iteratively hoisted outside the loop nest.898  for (auto *Loop : llvm::reverse(Loops)) {899    PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());900    Promoter.run(&TotalCountersPromoted);901  }902}903 904static bool needsRuntimeHookUnconditionally(const Triple &TT) {905  // On Fuchsia, we only need runtime hook if any counters are present.906  if (TT.isOSFuchsia())907    return false;908 909  return true;910}911 912/// Check if the module contains uses of any profiling intrinsics.913static bool containsProfilingIntrinsics(Module &M) {914  auto containsIntrinsic = [&](int ID) {915    if (auto *F = Intrinsic::getDeclarationIfExists(&M, ID))916      return !F->use_empty();917    return false;918  };919  return containsIntrinsic(Intrinsic::instrprof_cover) ||920         containsIntrinsic(Intrinsic::instrprof_increment) ||921         containsIntrinsic(Intrinsic::instrprof_increment_step) ||922         containsIntrinsic(Intrinsic::instrprof_timestamp) ||923         containsIntrinsic(Intrinsic::instrprof_value_profile);924}925 926bool InstrLowerer::lower() {927  bool MadeChange = false;928  bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);929  if (NeedsRuntimeHook)930    MadeChange = emitRuntimeHook();931 932  if (!IsCS && isSamplingEnabled())933    createProfileSamplingVar(M);934 935  bool ContainsProfiling = containsProfilingIntrinsics(M);936  GlobalVariable *CoverageNamesVar =937      M.getNamedGlobal(getCoverageUnusedNamesVarName());938  // Improve compile time by avoiding linear scans when there is no work.939  if (!ContainsProfiling && !CoverageNamesVar)940    return MadeChange;941 942  // We did not know how many value sites there would be inside943  // the instrumented function. This is counting the number of instrumented944  // target value sites to enter it as field in the profile data variable.945  for (Function &F : M) {946    InstrProfCntrInstBase *FirstProfInst = nullptr;947    for (BasicBlock &BB : F) {948      for (auto I = BB.begin(), E = BB.end(); I != E; I++) {949        if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))950          computeNumValueSiteCounts(Ind);951        else {952          if (FirstProfInst == nullptr &&953              (isa<InstrProfIncrementInst>(I) || isa<InstrProfCoverInst>(I)))954            FirstProfInst = dyn_cast<InstrProfCntrInstBase>(I);955          // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.956          if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(I))957            static_cast<void>(getOrCreateRegionBitmaps(Params));958        }959      }960    }961 962    // Use a profile intrinsic to create the region counters and data variable.963    // Also create the data variable based on the MCDCParams.964    if (FirstProfInst != nullptr) {965      static_cast<void>(getOrCreateRegionCounters(FirstProfInst));966    }967  }968 969  if (EnableVTableValueProfiling)970    for (GlobalVariable &GV : M.globals())971      // Global variables with type metadata are virtual table variables.972      if (GV.hasMetadata(LLVMContext::MD_type))973        getOrCreateVTableProfData(&GV);974 975  for (Function &F : M)976    MadeChange |= lowerIntrinsics(&F);977 978  if (CoverageNamesVar) {979    lowerCoverageData(CoverageNamesVar);980    MadeChange = true;981  }982 983  if (!MadeChange)984    return false;985 986  emitVNodes();987  emitNameData();988  emitVTableNames();989 990  // Emit runtime hook for the cases where the target does not unconditionally991  // require pulling in profile runtime, and coverage is enabled on code that is992  // not eliminated by the front-end, e.g. unused functions with internal993  // linkage.994  if (!NeedsRuntimeHook && ContainsProfiling)995    emitRuntimeHook();996 997  emitRegistration();998  emitUses();999  emitInitialization();1000  return true;1001}1002 1003static FunctionCallee getOrInsertValueProfilingCall(1004    Module &M, const TargetLibraryInfo &TLI,1005    ValueProfilingCallType CallType = ValueProfilingCallType::Default) {1006  LLVMContext &Ctx = M.getContext();1007  auto *ReturnTy = Type::getVoidTy(M.getContext());1008 1009  AttributeList AL;1010  if (auto AK = TLI.getExtAttrForI32Param(false))1011    AL = AL.addParamAttribute(M.getContext(), 2, AK);1012 1013  assert((CallType == ValueProfilingCallType::Default ||1014          CallType == ValueProfilingCallType::MemOp) &&1015         "Must be Default or MemOp");1016  Type *ParamTypes[] = {1017#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType1018#include "llvm/ProfileData/InstrProfData.inc"1019  };1020  auto *ValueProfilingCallTy =1021      FunctionType::get(ReturnTy, ArrayRef(ParamTypes), false);1022  StringRef FuncName = CallType == ValueProfilingCallType::Default1023                           ? getInstrProfValueProfFuncName()1024                           : getInstrProfValueProfMemOpFuncName();1025  return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);1026}1027 1028void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {1029  GlobalVariable *Name = Ind->getName();1030  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();1031  uint64_t Index = Ind->getIndex()->getZExtValue();1032  auto &PD = ProfileDataMap[Name];1033  PD.NumValueSites[ValueKind] =1034      std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));1035}1036 1037void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {1038  // TODO: Value profiling heavily depends on the data section which is omitted1039  // in lightweight mode. We need to move the value profile pointer to the1040  // Counter struct to get this working.1041  assert(1042      ProfileCorrelate == InstrProfCorrelator::NONE &&1043      "Value profiling is not yet supported with lightweight instrumentation");1044  GlobalVariable *Name = Ind->getName();1045  auto It = ProfileDataMap.find(Name);1046  assert(It != ProfileDataMap.end() && It->second.DataVar &&1047         "value profiling detected in function with no counter increment");1048 1049  GlobalVariable *DataVar = It->second.DataVar;1050  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();1051  uint64_t Index = Ind->getIndex()->getZExtValue();1052  for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)1053    Index += It->second.NumValueSites[Kind];1054 1055  IRBuilder<> Builder(Ind);1056  bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==1057                      llvm::InstrProfValueKind::IPVK_MemOPSize);1058  CallInst *Call = nullptr;1059  auto *TLI = &GetTLI(*Ind->getFunction());1060  auto *NormalizedDataVarPtr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(1061      DataVar, PointerType::get(M.getContext(), 0));1062 1063  // To support value profiling calls within Windows exception handlers, funclet1064  // information contained within operand bundles needs to be copied over to1065  // the library call. This is required for the IR to be processed by the1066  // WinEHPrepare pass.1067  SmallVector<OperandBundleDef, 1> OpBundles;1068  Ind->getOperandBundlesAsDefs(OpBundles);1069  if (!IsMemOpSize) {1070    Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,1071                      Builder.getInt32(Index)};1072    Call = Builder.CreateCall(getOrInsertValueProfilingCall(M, *TLI), Args,1073                              OpBundles);1074  } else {1075    Value *Args[3] = {Ind->getTargetValue(), NormalizedDataVarPtr,1076                      Builder.getInt32(Index)};1077    Call = Builder.CreateCall(1078        getOrInsertValueProfilingCall(M, *TLI, ValueProfilingCallType::MemOp),1079        Args, OpBundles);1080  }1081  if (auto AK = TLI->getExtAttrForI32Param(false))1082    Call->addParamAttr(2, AK);1083  Ind->replaceAllUsesWith(Call);1084  Ind->eraseFromParent();1085}1086 1087GlobalVariable *InstrLowerer::getOrCreateBiasVar(StringRef VarName) {1088  GlobalVariable *Bias = M.getGlobalVariable(VarName);1089  if (Bias)1090    return Bias;1091 1092  Type *Int64Ty = Type::getInt64Ty(M.getContext());1093 1094  // Compiler must define this variable when runtime counter relocation1095  // is being used. Runtime has a weak external reference that is used1096  // to check whether that's the case or not.1097  Bias = new GlobalVariable(M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,1098                            Constant::getNullValue(Int64Ty), VarName);1099  Bias->setVisibility(GlobalVariable::HiddenVisibility);1100  // A definition that's weak (linkonce_odr) without being in a COMDAT1101  // section wouldn't lead to link errors, but it would lead to a dead1102  // data word from every TU but one. Putting it in COMDAT ensures there1103  // will be exactly one data slot in the link.1104  if (TT.supportsCOMDAT())1105    Bias->setComdat(M.getOrInsertComdat(VarName));1106 1107  return Bias;1108}1109 1110Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {1111  auto *Counters = getOrCreateRegionCounters(I);1112  IRBuilder<> Builder(I);1113 1114  if (isa<InstrProfTimestampInst>(I))1115    Counters->setAlignment(Align(8));1116 1117  auto *Addr = Builder.CreateConstInBoundsGEP2_32(1118      Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());1119 1120  if (!isRuntimeCounterRelocationEnabled())1121    return Addr;1122 1123  Type *Int64Ty = Type::getInt64Ty(M.getContext());1124  Function *Fn = I->getParent()->getParent();1125  LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];1126  if (!BiasLI) {1127    IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());1128    auto *Bias = getOrCreateBiasVar(getInstrProfCounterBiasVarName());1129    BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias, "profc_bias");1130    // Bias doesn't change after startup.1131    BiasLI->setMetadata(LLVMContext::MD_invariant_load,1132                        MDNode::get(M.getContext(), {}));1133  }1134  auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);1135  return Builder.CreateIntToPtr(Add, Addr->getType());1136}1137 1138Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {1139  auto *Bitmaps = getOrCreateRegionBitmaps(I);1140  if (!isRuntimeCounterRelocationEnabled())1141    return Bitmaps;1142 1143  // Put BiasLI onto the entry block.1144  Type *Int64Ty = Type::getInt64Ty(M.getContext());1145  Function *Fn = I->getFunction();1146  IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());1147  auto *Bias = getOrCreateBiasVar(getInstrProfBitmapBiasVarName());1148  auto *BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias, "profbm_bias");1149  // Assume BiasLI invariant (in the function at least)1150  BiasLI->setMetadata(LLVMContext::MD_invariant_load,1151                      MDNode::get(M.getContext(), {}));1152 1153  // Add Bias to Bitmaps and put it before the intrinsic.1154  IRBuilder<> Builder(I);1155  return Builder.CreatePtrAdd(Bitmaps, BiasLI, "profbm_addr");1156}1157 1158void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {1159  auto *Addr = getCounterAddress(CoverInstruction);1160  IRBuilder<> Builder(CoverInstruction);1161  if (ConditionalCounterUpdate) {1162    Instruction *SplitBefore = CoverInstruction->getNextNode();1163    auto &Ctx = CoverInstruction->getParent()->getContext();1164    auto *Int8Ty = llvm::Type::getInt8Ty(Ctx);1165    Value *Load = Builder.CreateLoad(Int8Ty, Addr, "pgocount");1166    Value *Cmp = Builder.CreateIsNotNull(Load, "pgocount.ifnonzero");1167    Instruction *ThenBranch =1168        SplitBlockAndInsertIfThen(Cmp, SplitBefore, false);1169    Builder.SetInsertPoint(ThenBranch);1170  }1171 1172  // We store zero to represent that this block is covered.1173  Builder.CreateStore(Builder.getInt8(0), Addr);1174  CoverInstruction->eraseFromParent();1175}1176 1177void InstrLowerer::lowerTimestamp(1178    InstrProfTimestampInst *TimestampInstruction) {1179  assert(TimestampInstruction->getIndex()->isZeroValue() &&1180         "timestamp probes are always the first probe for a function");1181  auto &Ctx = M.getContext();1182  auto *TimestampAddr = getCounterAddress(TimestampInstruction);1183  IRBuilder<> Builder(TimestampInstruction);1184  auto *CalleeTy =1185      FunctionType::get(Type::getVoidTy(Ctx), TimestampAddr->getType(), false);1186  auto Callee = M.getOrInsertFunction(1187      INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy);1188  Builder.CreateCall(Callee, {TimestampAddr});1189  TimestampInstruction->eraseFromParent();1190}1191 1192void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {1193  auto *Addr = getCounterAddress(Inc);1194 1195  IRBuilder<> Builder(Inc);1196  if (Options.Atomic || AtomicCounterUpdateAll ||1197      (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {1198    Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),1199                            MaybeAlign(), AtomicOrdering::Monotonic);1200  } else {1201    Value *IncStep = Inc->getStep();1202    Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");1203    auto *Count = Builder.CreateAdd(Load, Inc->getStep());1204    auto *Store = Builder.CreateStore(Count, Addr);1205    if (isCounterPromotionEnabled())1206      PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);1207  }1208  Inc->eraseFromParent();1209}1210 1211void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {1212  ConstantArray *Names =1213      cast<ConstantArray>(CoverageNamesVar->getInitializer());1214  for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {1215    Constant *NC = Names->getOperand(I);1216    Value *V = NC->stripPointerCasts();1217    assert(isa<GlobalVariable>(V) && "Missing reference to function name");1218    GlobalVariable *Name = cast<GlobalVariable>(V);1219 1220    Name->setLinkage(GlobalValue::PrivateLinkage);1221    ReferencedNames.push_back(Name);1222    if (isa<ConstantExpr>(NC))1223      NC->dropAllReferences();1224  }1225  CoverageNamesVar->eraseFromParent();1226}1227 1228void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(1229    InstrProfMCDCTVBitmapUpdate *Update) {1230  auto &Ctx = M.getContext();1231  IRBuilder<> Builder(Update);1232  auto *Int8Ty = Type::getInt8Ty(Ctx);1233  auto *Int32Ty = Type::getInt32Ty(Ctx);1234  auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();1235  auto *BitmapAddr = getBitmapAddress(Update);1236 1237  // Load Temp Val + BitmapIdx.1238  //  %mcdc.temp = load i32, ptr %mcdc.addr, align 41239  auto *Temp = Builder.CreateAdd(1240      Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp"),1241      Update->getBitmapIndex());1242 1243  // Calculate byte offset using div8.1244  //  %1 = lshr i32 %mcdc.temp, 31245  auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);1246 1247  // Add byte offset to section base byte address.1248  // %4 = getelementptr inbounds i8, ptr @__profbm_test, i32 %11249  auto *BitmapByteAddr =1250      Builder.CreateInBoundsPtrAdd(BitmapAddr, BitmapByteOffset);1251 1252  // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)1253  //  %5 = and i32 %mcdc.temp, 71254  //  %6 = trunc i32 %5 to i81255  auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty);1256 1257  // Shift bit offset left to form a bitmap.1258  //  %7 = shl i8 1, %61259  auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);1260 1261  // Load profile bitmap byte.1262  //  %mcdc.bits = load i8, ptr %4, align 11263  auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits");1264 1265  if (Options.Atomic || AtomicCounterUpdateAll) {1266    // If ((Bitmap & Val) != Val), then execute atomic (Bitmap |= Val).1267    // Note, just-loaded Bitmap might not be up-to-date. Use it just for1268    // early testing.1269    auto *Masked = Builder.CreateAnd(Bitmap, ShiftedVal);1270    auto *ShouldStore = Builder.CreateICmpNE(Masked, ShiftedVal);1271 1272    // Assume updating will be rare.1273    auto *Unlikely = MDBuilder(Ctx).createUnlikelyBranchWeights();1274    Instruction *ThenBranch =1275        SplitBlockAndInsertIfThen(ShouldStore, Update, false, Unlikely);1276 1277    // Execute if (unlikely(ShouldStore)).1278    Builder.SetInsertPoint(ThenBranch);1279    Builder.CreateAtomicRMW(AtomicRMWInst::Or, BitmapByteAddr, ShiftedVal,1280                            MaybeAlign(), AtomicOrdering::Monotonic);1281  } else {1282    // Perform logical OR of profile bitmap byte and shifted bit offset.1283    //  %8 = or i8 %mcdc.bits, %71284    auto *Result = Builder.CreateOr(Bitmap, ShiftedVal);1285 1286    // Store the updated profile bitmap byte.1287    //  store i8 %8, ptr %3, align 11288    Builder.CreateStore(Result, BitmapByteAddr);1289  }1290 1291  Update->eraseFromParent();1292}1293 1294/// Get the name of a profiling variable for a particular function.1295static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,1296                              bool &Renamed) {1297  StringRef NamePrefix = getInstrProfNameVarPrefix();1298  StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());1299  Function *F = Inc->getParent()->getParent();1300  Module *M = F->getParent();1301  if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||1302      !canRenameComdatFunc(*F)) {1303    Renamed = false;1304    return (Prefix + Name).str();1305  }1306  Renamed = true;1307  uint64_t FuncHash = Inc->getHash()->getZExtValue();1308  SmallVector<char, 24> HashPostfix;1309  if (Name.ends_with((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))1310    return (Prefix + Name).str();1311  return (Prefix + Name + "." + Twine(FuncHash)).str();1312}1313 1314static inline bool shouldRecordFunctionAddr(Function *F) {1315  // Only record function addresses if IR PGO is enabled or if clang value1316  // profiling is enabled. Recording function addresses greatly increases object1317  // file size, because it prevents the inliner from deleting functions that1318  // have been inlined everywhere.1319  if (!profDataReferencedByCode(*F->getParent()))1320    return false;1321 1322  // Check the linkage1323  bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();1324  if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&1325      !HasAvailableExternallyLinkage)1326    return true;1327 1328  // A function marked 'alwaysinline' with available_externally linkage can't1329  // have its address taken. Doing so would create an undefined external ref to1330  // the function, which would fail to link.1331  if (HasAvailableExternallyLinkage &&1332      F->hasFnAttribute(Attribute::AlwaysInline))1333    return false;1334 1335  // Prohibit function address recording if the function is both internal and1336  // COMDAT. This avoids the profile data variable referencing internal symbols1337  // in COMDAT.1338  if (F->hasLocalLinkage() && F->hasComdat())1339    return false;1340 1341  // Check uses of this function for other than direct calls or invokes to it.1342  // Inline virtual functions have linkeOnceODR linkage. When a key method1343  // exists, the vtable will only be emitted in the TU where the key method1344  // is defined. In a TU where vtable is not available, the function won't1345  // be 'addresstaken'. If its address is not recorded here, the profile data1346  // with missing address may be picked by the linker leading  to missing1347  // indirect call target info.1348  return F->hasAddressTaken() || F->hasLinkOnceLinkage();1349}1350 1351static inline bool shouldUsePublicSymbol(Function *Fn) {1352  // It isn't legal to make an alias of this function at all1353  if (Fn->isDeclarationForLinker())1354    return true;1355 1356  // Symbols with local linkage can just use the symbol directly without1357  // introducing relocations1358  if (Fn->hasLocalLinkage())1359    return true;1360 1361  // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some1362  // unfavorable interaction between the new alias and the alias renaming done1363  // in LowerTypeTests under ThinLTO. For comdat functions that would normally1364  // be deduplicated, but the renaming scheme ends up preventing renaming, since1365  // it creates unique names for each alias, resulting in duplicated symbols. In1366  // the future, we should update the CFI related passes to migrate these1367  // aliases to the same module as the jump-table they refer to will be defined.1368  if (Fn->hasMetadata(LLVMContext::MD_type))1369    return true;1370 1371  // For comdat functions, an alias would need the same linkage as the original1372  // function and hidden visibility. There is no point in adding an alias with1373  // identical linkage an visibility to avoid introducing symbolic relocations.1374  if (Fn->hasComdat() &&1375      (Fn->getVisibility() == GlobalValue::VisibilityTypes::HiddenVisibility))1376    return true;1377 1378  // its OK to use an alias1379  return false;1380}1381 1382static inline Constant *getFuncAddrForProfData(Function *Fn) {1383  auto *Int8PtrTy = PointerType::getUnqual(Fn->getContext());1384  // Store a nullptr in __llvm_profd, if we shouldn't use a real address1385  if (!shouldRecordFunctionAddr(Fn))1386    return ConstantPointerNull::get(Int8PtrTy);1387 1388  // If we can't use an alias, we must use the public symbol, even though this1389  // may require a symbolic relocation.1390  if (shouldUsePublicSymbol(Fn))1391    return Fn;1392 1393  // When possible use a private alias to avoid symbolic relocations.1394  auto *GA = GlobalAlias::create(GlobalValue::LinkageTypes::PrivateLinkage,1395                                 Fn->getName() + ".local", Fn);1396 1397  // When the instrumented function is a COMDAT function, we cannot use a1398  // private alias. If we did, we would create reference to a local label in1399  // this function's section. If this version of the function isn't selected by1400  // the linker, then the metadata would introduce a reference to a discarded1401  // section. So, for COMDAT functions, we need to adjust the linkage of the1402  // alias. Using hidden visibility avoids a dynamic relocation and an entry in1403  // the dynamic symbol table.1404  //1405  // Note that this handles COMDAT functions with visibility other than Hidden,1406  // since that case is covered in shouldUsePublicSymbol()1407  if (Fn->hasComdat()) {1408    GA->setLinkage(Fn->getLinkage());1409    GA->setVisibility(GlobalValue::VisibilityTypes::HiddenVisibility);1410  }1411 1412  // appendToCompilerUsed(*Fn->getParent(), {GA});1413 1414  return GA;1415}1416 1417static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {1418  // compiler-rt uses linker support to get data/counters/name start/end for1419  // ELF, COFF, Mach-O, XCOFF, and Wasm.1420  if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||1421      TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF() ||1422      TT.isOSBinFormatWasm())1423    return false;1424 1425  return true;1426}1427 1428void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO,1429                                  StringRef CounterGroupName) {1430  // Place lowered global variables in a comdat group if the associated function1431  // or global variable is a COMDAT. This will make sure that only one copy of1432  // global variable (e.g. function counters) of the COMDAT function will be1433  // emitted after linking.1434  bool NeedComdat = needsComdatForCounter(*GO, M);1435  bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());1436 1437  if (!UseComdat)1438    return;1439 1440  // Keep in mind that this pass may run before the inliner, so we need to1441  // create a new comdat group (for counters, profiling data, etc). If we use1442  // the comdat of the parent function, that will result in relocations against1443  // discarded sections.1444  //1445  // If the data variable is referenced by code, non-counter variables (notably1446  // profiling data) and counters have to be in different comdats for COFF1447  // because the Visual C++ linker will report duplicate symbol errors if there1448  // are multiple external symbols with the same name marked1449  // IMAGE_COMDAT_SELECT_ASSOCIATIVE.1450  StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode1451                            ? GV->getName()1452                            : CounterGroupName;1453  Comdat *C = M.getOrInsertComdat(GroupName);1454 1455  if (!NeedComdat) {1456    // Object file format must be ELF since `UseComdat && !NeedComdat` is true.1457    //1458    // For ELF, when not using COMDAT, put counters, data and values into a1459    // nodeduplicate COMDAT which is lowered to a zero-flag section group. This1460    // allows -z start-stop-gc to discard the entire group when the function is1461    // discarded.1462    C->setSelectionKind(Comdat::NoDeduplicate);1463  }1464  GV->setComdat(C);1465  // COFF doesn't allow the comdat group leader to have private linkage, so1466  // upgrade private linkage to internal linkage to produce a symbol table1467  // entry.1468  if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())1469    GV->setLinkage(GlobalValue::InternalLinkage);1470}1471 1472static inline bool shouldRecordVTableAddr(GlobalVariable *GV) {1473  if (!profDataReferencedByCode(*GV->getParent()))1474    return false;1475 1476  if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&1477      !GV->hasAvailableExternallyLinkage())1478    return true;1479 1480  // This avoids the profile data from referencing internal symbols in1481  // COMDAT.1482  if (GV->hasLocalLinkage() && GV->hasComdat())1483    return false;1484 1485  return true;1486}1487 1488// FIXME: Introduce an internal alias like what's done for functions to reduce1489// the number of relocation entries.1490static inline Constant *getVTableAddrForProfData(GlobalVariable *GV) {1491  // Store a nullptr in __profvt_ if a real address shouldn't be used.1492  if (!shouldRecordVTableAddr(GV))1493    return ConstantPointerNull::get(PointerType::getUnqual(GV->getContext()));1494 1495  return GV;1496}1497 1498void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) {1499  assert(ProfileCorrelate != InstrProfCorrelator::DEBUG_INFO &&1500         "Value profiling is not supported with lightweight instrumentation");1501  if (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())1502    return;1503 1504  // Skip llvm internal global variable or __prof variables.1505  if (GV->getName().starts_with("llvm.") ||1506      GV->getName().starts_with("__llvm") ||1507      GV->getName().starts_with("__prof"))1508    return;1509 1510  // VTableProfData already created1511  auto It = VTableDataMap.find(GV);1512  if (It != VTableDataMap.end() && It->second)1513    return;1514 1515  GlobalValue::LinkageTypes Linkage = GV->getLinkage();1516  GlobalValue::VisibilityTypes Visibility = GV->getVisibility();1517 1518  // This is to keep consistent with per-function profile data1519  // for correctness.1520  if (TT.isOSBinFormatXCOFF()) {1521    Linkage = GlobalValue::InternalLinkage;1522    Visibility = GlobalValue::DefaultVisibility;1523  }1524 1525  LLVMContext &Ctx = M.getContext();1526  Type *DataTypes[] = {1527#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,1528#include "llvm/ProfileData/InstrProfData.inc"1529#undef INSTR_PROF_VTABLE_DATA1530  };1531 1532  auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));1533 1534  // Used by INSTR_PROF_VTABLE_DATA MACRO1535  Constant *VTableAddr = getVTableAddrForProfData(GV);1536  const std::string PGOVTableName = getPGOName(*GV);1537  // Record the length of the vtable. This is needed since vtable pointers1538  // loaded from C++ objects might be from the middle of a vtable definition.1539  uint32_t VTableSizeVal =1540      M.getDataLayout().getTypeAllocSize(GV->getValueType());1541 1542  Constant *DataVals[] = {1543#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,1544#include "llvm/ProfileData/InstrProfData.inc"1545#undef INSTR_PROF_VTABLE_DATA1546  };1547 1548  auto *Data =1549      new GlobalVariable(M, DataTy, /*constant=*/false, Linkage,1550                         ConstantStruct::get(DataTy, DataVals),1551                         getInstrProfVTableVarPrefix() + PGOVTableName);1552 1553  Data->setVisibility(Visibility);1554  Data->setSection(getInstrProfSectionName(IPSK_vtab, TT.getObjectFormat()));1555  Data->setAlignment(Align(8));1556 1557  maybeSetComdat(Data, GV, Data->getName());1558 1559  VTableDataMap[GV] = Data;1560 1561  ReferencedVTables.push_back(GV);1562 1563  // VTable <Hash, Addr> is used by runtime but not referenced by other1564  // sections. Conservatively mark it linker retained.1565  UsedVars.push_back(Data);1566}1567 1568GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,1569                                                  InstrProfSectKind IPSK) {1570  GlobalVariable *NamePtr = Inc->getName();1571 1572  // Match the linkage and visibility of the name global.1573  Function *Fn = Inc->getParent()->getParent();1574  GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();1575  GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();1576 1577  // Use internal rather than private linkage so the counter variable shows up1578  // in the symbol table when using debug info for correlation.1579  if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO &&1580      TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage)1581    Linkage = GlobalValue::InternalLinkage;1582 1583  // Due to the limitation of binder as of 2021/09/28, the duplicate weak1584  // symbols in the same csect won't be discarded. When there are duplicate weak1585  // symbols, we can NOT guarantee that the relocations get resolved to the1586  // intended weak symbol, so we can not ensure the correctness of the relative1587  // CounterPtr, so we have to use private linkage for counter and data symbols.1588  if (TT.isOSBinFormatXCOFF()) {1589    Linkage = GlobalValue::PrivateLinkage;1590    Visibility = GlobalValue::DefaultVisibility;1591  }1592  // Move the name variable to the right section.1593  bool Renamed;1594  GlobalVariable *Ptr;1595  StringRef VarPrefix;1596  std::string VarName;1597  if (IPSK == IPSK_cnts) {1598    VarPrefix = getInstrProfCountersVarPrefix();1599    VarName = getVarName(Inc, VarPrefix, Renamed);1600    InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Inc);1601    Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);1602  } else if (IPSK == IPSK_bitmap) {1603    VarPrefix = getInstrProfBitmapVarPrefix();1604    VarName = getVarName(Inc, VarPrefix, Renamed);1605    InstrProfMCDCBitmapInstBase *BitmapUpdate =1606        dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);1607    Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);1608  } else {1609    llvm_unreachable("Profile Section must be for Counters or Bitmaps");1610  }1611 1612  Ptr->setVisibility(Visibility);1613  // Put the counters and bitmaps in their own sections so linkers can1614  // remove unneeded sections.1615  Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat()));1616  Ptr->setLinkage(Linkage);1617  maybeSetComdat(Ptr, Fn, VarName);1618  return Ptr;1619}1620 1621GlobalVariable *1622InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,1623                                  StringRef Name,1624                                  GlobalValue::LinkageTypes Linkage) {1625  uint64_t NumBytes = Inc->getNumBitmapBytes();1626  auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M.getContext()), NumBytes);1627  auto GV = new GlobalVariable(M, BitmapTy, false, Linkage,1628                               Constant::getNullValue(BitmapTy), Name);1629  GV->setAlignment(Align(1));1630  return GV;1631}1632 1633GlobalVariable *1634InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {1635  GlobalVariable *NamePtr = Inc->getName();1636  auto &PD = ProfileDataMap[NamePtr];1637  if (PD.RegionBitmaps)1638    return PD.RegionBitmaps;1639 1640  // If RegionBitmaps doesn't already exist, create it by first setting up1641  // the corresponding profile section.1642  auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);1643  PD.RegionBitmaps = BitmapPtr;1644  PD.NumBitmapBytes = Inc->getNumBitmapBytes();1645  return PD.RegionBitmaps;1646}1647 1648GlobalVariable *1649InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,1650                                   GlobalValue::LinkageTypes Linkage) {1651  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();1652  auto &Ctx = M.getContext();1653  GlobalVariable *GV;1654  if (isa<InstrProfCoverInst>(Inc)) {1655    auto *CounterTy = Type::getInt8Ty(Ctx);1656    auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);1657    // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.1658    std::vector<Constant *> InitialValues(NumCounters,1659                                          Constant::getAllOnesValue(CounterTy));1660    GV = new GlobalVariable(M, CounterArrTy, false, Linkage,1661                            ConstantArray::get(CounterArrTy, InitialValues),1662                            Name);1663    GV->setAlignment(Align(1));1664  } else {1665    auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);1666    GV = new GlobalVariable(M, CounterTy, false, Linkage,1667                            Constant::getNullValue(CounterTy), Name);1668    GV->setAlignment(Align(8));1669  }1670  return GV;1671}1672 1673GlobalVariable *1674InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {1675  GlobalVariable *NamePtr = Inc->getName();1676  auto &PD = ProfileDataMap[NamePtr];1677  if (PD.RegionCounters)1678    return PD.RegionCounters;1679 1680  // If RegionCounters doesn't already exist, create it by first setting up1681  // the corresponding profile section.1682  auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);1683  PD.RegionCounters = CounterPtr;1684 1685  if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO) {1686    LLVMContext &Ctx = M.getContext();1687    Function *Fn = Inc->getParent()->getParent();1688    if (auto *SP = Fn->getSubprogram()) {1689      DIBuilder DB(M, true, SP->getUnit());1690      Metadata *FunctionNameAnnotation[] = {1691          MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName),1692          MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)),1693      };1694      Metadata *CFGHashAnnotation[] = {1695          MDString::get(Ctx, InstrProfCorrelator::CFGHashAttributeName),1696          ConstantAsMetadata::get(Inc->getHash()),1697      };1698      Metadata *NumCountersAnnotation[] = {1699          MDString::get(Ctx, InstrProfCorrelator::NumCountersAttributeName),1700          ConstantAsMetadata::get(Inc->getNumCounters()),1701      };1702      auto Annotations = DB.getOrCreateArray({1703          MDNode::get(Ctx, FunctionNameAnnotation),1704          MDNode::get(Ctx, CFGHashAnnotation),1705          MDNode::get(Ctx, NumCountersAnnotation),1706      });1707      auto *DICounter = DB.createGlobalVariableExpression(1708          SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),1709          /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),1710          CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,1711          /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,1712          Annotations);1713      CounterPtr->addDebugInfo(DICounter);1714      DB.finalize();1715    }1716 1717    // Mark the counter variable as used so that it isn't optimized out.1718    CompilerUsedVars.push_back(PD.RegionCounters);1719  }1720 1721  // Create the data variable (if it doesn't already exist).1722  createDataVariable(Inc);1723 1724  return PD.RegionCounters;1725}1726 1727void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {1728  // When debug information is correlated to profile data, a data variable1729  // is not needed.1730  if (ProfileCorrelate == InstrProfCorrelator::DEBUG_INFO)1731    return;1732 1733  GlobalVariable *NamePtr = Inc->getName();1734  auto &PD = ProfileDataMap[NamePtr];1735 1736  // Return if data variable was already created.1737  if (PD.DataVar)1738    return;1739 1740  LLVMContext &Ctx = M.getContext();1741 1742  Function *Fn = Inc->getParent()->getParent();1743  GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();1744  GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();1745 1746  // Due to the limitation of binder as of 2021/09/28, the duplicate weak1747  // symbols in the same csect won't be discarded. When there are duplicate weak1748  // symbols, we can NOT guarantee that the relocations get resolved to the1749  // intended weak symbol, so we can not ensure the correctness of the relative1750  // CounterPtr, so we have to use private linkage for counter and data symbols.1751  if (TT.isOSBinFormatXCOFF()) {1752    Linkage = GlobalValue::PrivateLinkage;1753    Visibility = GlobalValue::DefaultVisibility;1754  }1755 1756  bool NeedComdat = needsComdatForCounter(*Fn, M);1757  bool Renamed;1758 1759  // The Data Variable section is anchored to profile counters.1760  std::string CntsVarName =1761      getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);1762  std::string DataVarName =1763      getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);1764 1765  auto *Int8PtrTy = PointerType::getUnqual(Ctx);1766  // Allocate statically the array of pointers to value profile nodes for1767  // the current function.1768  Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);1769  uint64_t NS = 0;1770  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)1771    NS += PD.NumValueSites[Kind];1772  if (NS > 0 && ValueProfileStaticAlloc &&1773      !needsRuntimeRegistrationOfSectionRange(TT)) {1774    ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);1775    auto *ValuesVar = new GlobalVariable(1776        M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),1777        getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));1778    ValuesVar->setVisibility(Visibility);1779    setGlobalVariableLargeSection(TT, *ValuesVar);1780    ValuesVar->setSection(1781        getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));1782    ValuesVar->setAlignment(Align(8));1783    maybeSetComdat(ValuesVar, Fn, CntsVarName);1784    ValuesPtrExpr = ConstantExpr::getPointerBitCastOrAddrSpaceCast(1785        ValuesVar, PointerType::get(Fn->getContext(), 0));1786  }1787 1788  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();1789  auto *CounterPtr = PD.RegionCounters;1790 1791  uint64_t NumBitmapBytes = PD.NumBitmapBytes;1792 1793  // Create data variable.1794  auto *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext());1795  auto *Int16Ty = Type::getInt16Ty(Ctx);1796  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);1797  Type *DataTypes[] = {1798#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,1799#include "llvm/ProfileData/InstrProfData.inc"1800  };1801  auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));1802 1803  Constant *FunctionAddr = getFuncAddrForProfData(Fn);1804 1805  Constant *Int16ArrayVals[IPVK_Last + 1];1806  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)1807    Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);1808 1809  if (isGPUProfTarget(M)) {1810    Linkage = GlobalValue::ExternalLinkage;1811    Visibility = GlobalValue::ProtectedVisibility;1812  }1813  // If the data variable is not referenced by code (if we don't emit1814  // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the1815  // data variable live under linker GC, the data variable can be private. This1816  // optimization applies to ELF.1817  //1818  // On COFF, a comdat leader cannot be local so we require DataReferencedByCode1819  // to be false.1820  //1821  // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees1822  // that other copies must have the same CFG and cannot have value profiling.1823  // If no hash suffix, other profd copies may be referenced by code.1824  else if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&1825           (TT.isOSBinFormatELF() ||1826            (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {1827    Linkage = GlobalValue::PrivateLinkage;1828    Visibility = GlobalValue::DefaultVisibility;1829  }1830  auto *Data =1831      new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName);1832  Constant *RelativeCounterPtr;1833  GlobalVariable *BitmapPtr = PD.RegionBitmaps;1834  Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);1835  InstrProfSectKind DataSectionKind;1836  // With binary profile correlation, profile data is not loaded into memory.1837  // profile data must reference profile counter with an absolute relocation.1838  if (ProfileCorrelate == InstrProfCorrelator::BINARY) {1839    DataSectionKind = IPSK_covdata;1840    RelativeCounterPtr = ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy);1841    if (BitmapPtr != nullptr)1842      RelativeBitmapPtr = ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy);1843  } else {1844    // Reference the counter variable with a label difference (link-time1845    // constant).1846    DataSectionKind = IPSK_data;1847    RelativeCounterPtr =1848        ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),1849                             ConstantExpr::getPtrToInt(Data, IntPtrTy));1850    if (BitmapPtr != nullptr)1851      RelativeBitmapPtr =1852          ConstantExpr::getSub(ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy),1853                               ConstantExpr::getPtrToInt(Data, IntPtrTy));1854  }1855 1856  Constant *DataVals[] = {1857#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,1858#include "llvm/ProfileData/InstrProfData.inc"1859  };1860  Data->setInitializer(ConstantStruct::get(DataTy, DataVals));1861 1862  Data->setVisibility(Visibility);1863  Data->setSection(1864      getInstrProfSectionName(DataSectionKind, TT.getObjectFormat()));1865  Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));1866  maybeSetComdat(Data, Fn, CntsVarName);1867 1868  PD.DataVar = Data;1869 1870  // Mark the data variable as used so that it isn't stripped out.1871  CompilerUsedVars.push_back(Data);1872  // Now that the linkage set by the FE has been passed to the data and counter1873  // variables, reset Name variable's linkage and visibility to private so that1874  // it can be removed later by the compiler.1875  NamePtr->setLinkage(GlobalValue::PrivateLinkage);1876  // Collect the referenced names to be used by emitNameData.1877  ReferencedNames.push_back(NamePtr);1878}1879 1880void InstrLowerer::emitVNodes() {1881  if (!ValueProfileStaticAlloc)1882    return;1883 1884  // For now only support this on platforms that do1885  // not require runtime registration to discover1886  // named section start/end.1887  if (needsRuntimeRegistrationOfSectionRange(TT))1888    return;1889 1890  size_t TotalNS = 0;1891  for (auto &PD : ProfileDataMap) {1892    for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)1893      TotalNS += PD.second.NumValueSites[Kind];1894  }1895 1896  if (!TotalNS)1897    return;1898 1899  uint64_t NumCounters = TotalNS * NumCountersPerValueSite;1900// Heuristic for small programs with very few total value sites.1901// The default value of vp-counters-per-site is chosen based on1902// the observation that large apps usually have a low percentage1903// of value sites that actually have any profile data, and thus1904// the average number of counters per site is low. For small1905// apps with very few sites, this may not be true. Bump up the1906// number of counters in this case.1907#define INSTR_PROF_MIN_VAL_COUNTS 101908  if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)1909    NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);1910 1911  auto &Ctx = M.getContext();1912  Type *VNodeTypes[] = {1913#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,1914#include "llvm/ProfileData/InstrProfData.inc"1915  };1916  auto *VNodeTy = StructType::get(Ctx, ArrayRef(VNodeTypes));1917 1918  ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);1919  auto *VNodesVar = new GlobalVariable(1920      M, VNodesTy, false, GlobalValue::PrivateLinkage,1921      Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());1922  setGlobalVariableLargeSection(TT, *VNodesVar);1923  VNodesVar->setSection(1924      getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));1925  VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(VNodesTy));1926  // VNodesVar is used by runtime but not referenced via relocation by other1927  // sections. Conservatively make it linker retained.1928  UsedVars.push_back(VNodesVar);1929}1930 1931void InstrLowerer::emitNameData() {1932  if (ReferencedNames.empty())1933    return;1934 1935  std::string CompressedNameStr;1936  if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,1937                                          DoInstrProfNameCompression)) {1938    report_fatal_error(Twine(toString(std::move(E))), false);1939  }1940 1941  auto &Ctx = M.getContext();1942  auto *NamesVal =1943      ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);1944  NamesVar = new GlobalVariable(M, NamesVal->getType(), true,1945                                GlobalValue::PrivateLinkage, NamesVal,1946                                getInstrProfNamesVarName());1947  if (isGPUProfTarget(M)) {1948    NamesVar->setLinkage(GlobalValue::ExternalLinkage);1949    NamesVar->setVisibility(GlobalValue::ProtectedVisibility);1950  }1951 1952  NamesSize = CompressedNameStr.size();1953  setGlobalVariableLargeSection(TT, *NamesVar);1954  NamesVar->setSection(1955      ProfileCorrelate == InstrProfCorrelator::BINARY1956          ? getInstrProfSectionName(IPSK_covname, TT.getObjectFormat())1957          : getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));1958  // On COFF, it's important to reduce the alignment down to 1 to prevent the1959  // linker from inserting padding before the start of the names section or1960  // between names entries.1961  NamesVar->setAlignment(Align(1));1962  // NamesVar is used by runtime but not referenced via relocation by other1963  // sections. Conservatively make it linker retained.1964  UsedVars.push_back(NamesVar);1965 1966  for (auto *NamePtr : ReferencedNames)1967    NamePtr->eraseFromParent();1968}1969 1970void InstrLowerer::emitVTableNames() {1971  if (!EnableVTableValueProfiling || ReferencedVTables.empty())1972    return;1973 1974  // Collect the PGO names of referenced vtables and compress them.1975  std::string CompressedVTableNames;1976  if (Error E = collectVTableStrings(ReferencedVTables, CompressedVTableNames,1977                                     DoInstrProfNameCompression)) {1978    report_fatal_error(Twine(toString(std::move(E))), false);1979  }1980 1981  auto &Ctx = M.getContext();1982  auto *VTableNamesVal = ConstantDataArray::getString(1983      Ctx, StringRef(CompressedVTableNames), false /* AddNull */);1984  GlobalVariable *VTableNamesVar =1985      new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */,1986                         GlobalValue::PrivateLinkage, VTableNamesVal,1987                         getInstrProfVTableNamesVarName());1988  VTableNamesVar->setSection(1989      getInstrProfSectionName(IPSK_vname, TT.getObjectFormat()));1990  VTableNamesVar->setAlignment(Align(1));1991  // Make VTableNames linker retained.1992  UsedVars.push_back(VTableNamesVar);1993}1994 1995void InstrLowerer::emitRegistration() {1996  if (!needsRuntimeRegistrationOfSectionRange(TT))1997    return;1998 1999  // Construct the function.2000  auto *VoidTy = Type::getVoidTy(M.getContext());2001  auto *VoidPtrTy = PointerType::getUnqual(M.getContext());2002  auto *Int64Ty = Type::getInt64Ty(M.getContext());2003  auto *RegisterFTy = FunctionType::get(VoidTy, false);2004  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,2005                                     getInstrProfRegFuncsName(), M);2006  RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);2007  if (Options.NoRedZone)2008    RegisterF->addFnAttr(Attribute::NoRedZone);2009 2010  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);2011  auto *RuntimeRegisterF =2012      Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,2013                       getInstrProfRegFuncName(), M);2014 2015  IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", RegisterF));2016  for (Value *Data : CompilerUsedVars)2017    if (!isa<Function>(Data))2018      // Check for addrspace cast when profiling GPU2019      IRB.CreateCall(RuntimeRegisterF,2020                     IRB.CreatePointerBitCastOrAddrSpaceCast(Data, VoidPtrTy));2021  for (Value *Data : UsedVars)2022    if (Data != NamesVar && !isa<Function>(Data))2023      IRB.CreateCall(RuntimeRegisterF,2024                     IRB.CreatePointerBitCastOrAddrSpaceCast(Data, VoidPtrTy));2025 2026  if (NamesVar) {2027    Type *ParamTypes[] = {VoidPtrTy, Int64Ty};2028    auto *NamesRegisterTy =2029        FunctionType::get(VoidTy, ArrayRef(ParamTypes), false);2030    auto *NamesRegisterF =2031        Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,2032                         getInstrProfNamesRegFuncName(), M);2033    IRB.CreateCall(NamesRegisterF, {IRB.CreatePointerBitCastOrAddrSpaceCast(2034                                        NamesVar, VoidPtrTy),2035                                    IRB.getInt64(NamesSize)});2036  }2037 2038  IRB.CreateRetVoid();2039}2040 2041bool InstrLowerer::emitRuntimeHook() {2042  // We expect the linker to be invoked with -u<hook_var> flag for Linux2043  // in which case there is no need to emit the external variable.2044  if (TT.isOSLinux() || TT.isOSAIX())2045    return false;2046 2047  // If the module's provided its own runtime, we don't need to do anything.2048  if (M.getGlobalVariable(getInstrProfRuntimeHookVarName()))2049    return false;2050 2051  // Declare an external variable that will pull in the runtime initialization.2052  auto *Int32Ty = Type::getInt32Ty(M.getContext());2053  auto *Var =2054      new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,2055                         nullptr, getInstrProfRuntimeHookVarName());2056  if (isGPUProfTarget(M))2057    Var->setVisibility(GlobalValue::ProtectedVisibility);2058  else2059    Var->setVisibility(GlobalValue::HiddenVisibility);2060 2061  if (TT.isOSBinFormatELF() && !TT.isPS()) {2062    // Mark the user variable as used so that it isn't stripped out.2063    CompilerUsedVars.push_back(Var);2064  } else {2065    // Make a function that uses it.2066    auto *User = Function::Create(FunctionType::get(Int32Ty, false),2067                                  GlobalValue::LinkOnceODRLinkage,2068                                  getInstrProfRuntimeHookVarUseFuncName(), M);2069    User->addFnAttr(Attribute::NoInline);2070    if (Options.NoRedZone)2071      User->addFnAttr(Attribute::NoRedZone);2072    User->setVisibility(GlobalValue::HiddenVisibility);2073    if (TT.supportsCOMDAT())2074      User->setComdat(M.getOrInsertComdat(User->getName()));2075 2076    IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", User));2077    auto *Load = IRB.CreateLoad(Int32Ty, Var);2078    IRB.CreateRet(Load);2079 2080    // Mark the function as used so that it isn't stripped out.2081    CompilerUsedVars.push_back(User);2082  }2083  return true;2084}2085 2086void InstrLowerer::emitUses() {2087  // The metadata sections are parallel arrays. Optimizers (e.g.2088  // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so2089  // we conservatively retain all unconditionally in the compiler.2090  //2091  // On ELF and Mach-O, the linker can guarantee the associated sections will be2092  // retained or discarded as a unit, so llvm.compiler.used is sufficient.2093  // Similarly on COFF, if prof data is not referenced by code we use one comdat2094  // and ensure this GC property as well. Otherwise, we have to conservatively2095  // make all of the sections retained by the linker.2096  if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||2097      (TT.isOSBinFormatCOFF() && !DataReferencedByCode))2098    appendToCompilerUsed(M, CompilerUsedVars);2099  else2100    appendToUsed(M, CompilerUsedVars);2101 2102  // We do not add proper references from used metadata sections to NamesVar and2103  // VNodesVar, so we have to be conservative and place them in llvm.used2104  // regardless of the target,2105  appendToUsed(M, UsedVars);2106}2107 2108void InstrLowerer::emitInitialization() {2109  // Create ProfileFileName variable. Don't don't this for the2110  // context-sensitive instrumentation lowering: This lowering is after2111  // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should2112  // have already create the variable before LTO/ThinLTO linking.2113  if (!IsCS)2114    createProfileFileNameVar(M, Options.InstrProfileOutput);2115  Function *RegisterF = M.getFunction(getInstrProfRegFuncsName());2116  if (!RegisterF)2117    return;2118 2119  // Create the initialization function.2120  auto *VoidTy = Type::getVoidTy(M.getContext());2121  auto *F = Function::Create(FunctionType::get(VoidTy, false),2122                             GlobalValue::InternalLinkage,2123                             getInstrProfInitFuncName(), M);2124  F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);2125  F->addFnAttr(Attribute::NoInline);2126  if (Options.NoRedZone)2127    F->addFnAttr(Attribute::NoRedZone);2128 2129  // Add the basic block and the necessary calls.2130  IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", F));2131  IRB.CreateCall(RegisterF, {});2132  IRB.CreateRetVoid();2133 2134  appendToGlobalCtors(M, F, 0);2135}2136 2137namespace llvm {2138// Create the variable for profile sampling.2139void createProfileSamplingVar(Module &M) {2140  const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));2141  IntegerType *SamplingVarTy;2142  Constant *ValueZero;2143  if (getSampledInstrumentationConfig().UseShort) {2144    SamplingVarTy = Type::getInt16Ty(M.getContext());2145    ValueZero = Constant::getIntegerValue(SamplingVarTy, APInt(16, 0));2146  } else {2147    SamplingVarTy = Type::getInt32Ty(M.getContext());2148    ValueZero = Constant::getIntegerValue(SamplingVarTy, APInt(32, 0));2149  }2150  auto SamplingVar = new GlobalVariable(2151      M, SamplingVarTy, false, GlobalValue::WeakAnyLinkage, ValueZero, VarName);2152  SamplingVar->setVisibility(GlobalValue::DefaultVisibility);2153  SamplingVar->setThreadLocal(true);2154  Triple TT(M.getTargetTriple());2155  if (TT.supportsCOMDAT()) {2156    SamplingVar->setLinkage(GlobalValue::ExternalLinkage);2157    SamplingVar->setComdat(M.getOrInsertComdat(VarName));2158  }2159  appendToCompilerUsed(M, SamplingVar);2160}2161} // namespace llvm2162