151 lines · cpp
1//===- PseudoProbeInserter.cpp - Insert annotation for callsite profiling -===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements PseudoProbeInserter pass, which inserts pseudo probe10// annotations for call instructions with a pseudo-probe-specific dwarf11// discriminator. such discriminator indicates that the call instruction comes12// with a pseudo probe, and the discriminator value holds information to13// identify the corresponding counter.14//===----------------------------------------------------------------------===//15 16#include "llvm/CodeGen/MachineBasicBlock.h"17#include "llvm/CodeGen/MachineFunctionPass.h"18#include "llvm/CodeGen/MachineInstr.h"19#include "llvm/CodeGen/TargetInstrInfo.h"20#include "llvm/IR/DebugInfoMetadata.h"21#include "llvm/IR/Module.h"22#include "llvm/IR/PseudoProbe.h"23#include "llvm/InitializePasses.h"24 25#define DEBUG_TYPE "pseudo-probe-inserter"26 27using namespace llvm;28 29namespace {30class PseudoProbeInserter : public MachineFunctionPass {31public:32 static char ID;33 34 PseudoProbeInserter() : MachineFunctionPass(ID) {35 initializePseudoProbeInserterPass(*PassRegistry::getPassRegistry());36 }37 38 StringRef getPassName() const override { return "Pseudo Probe Inserter"; }39 40 void getAnalysisUsage(AnalysisUsage &AU) const override {41 AU.setPreservesAll();42 MachineFunctionPass::getAnalysisUsage(AU);43 }44 45 bool doInitialization(Module &M) override {46 ShouldRun = M.getNamedMetadata(PseudoProbeDescMetadataName);47 return false;48 }49 50 bool runOnMachineFunction(MachineFunction &MF) override {51 if (!ShouldRun)52 return false;53 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();54 bool Changed = false;55 for (MachineBasicBlock &MBB : MF) {56 MachineInstr *FirstInstr = nullptr;57 for (MachineInstr &MI : MBB) {58 if (!MI.isPseudo())59 FirstInstr = &MI;60 if (MI.isCall()) {61 if (DILocation *DL = MI.getDebugLoc()) {62 auto Value = DL->getDiscriminator();63 if (DILocation::isPseudoProbeDiscriminator(Value)) {64 BuildMI(MBB, MI, DL, TII->get(TargetOpcode::PSEUDO_PROBE))65 .addImm(getFuncGUID(MF.getFunction().getParent(), DL))66 .addImm(67 PseudoProbeDwarfDiscriminator::extractProbeIndex(Value))68 .addImm(69 PseudoProbeDwarfDiscriminator::extractProbeType(Value))70 .addImm(PseudoProbeDwarfDiscriminator::extractProbeAttributes(71 Value));72 Changed = true;73 }74 }75 }76 }77 78 // Walk the block backwards, move PSEUDO_PROBE before the first real79 // instruction to fix out-of-order probes. There is a problem with probes80 // as the terminator of the block. During the offline counts processing,81 // the samples collected on the first physical instruction following a82 // probe will be counted towards the probe. This logically equals to83 // treating the instruction next to a probe as if it is from the same84 // block of the probe. This is accurate most of the time unless the85 // instruction can be reached from multiple flows, which means it actually86 // starts a new block. Samples collected on such probes may cause87 // imprecision with the counts inference algorithm. Fortunately, if88 // there are still other native instructions preceding the probe we can89 // use them as a place holder to collect samples for the probe.90 if (FirstInstr) {91 auto MII = MBB.rbegin();92 while (MII != MBB.rend()) {93 // Skip all pseudo probes followed by a real instruction since they94 // are not dangling.95 if (!MII->isPseudo())96 break;97 auto Cur = MII++;98 if (Cur->getOpcode() != TargetOpcode::PSEUDO_PROBE)99 continue;100 // Move the dangling probe before FirstInstr.101 auto *ProbeInstr = &*Cur;102 MBB.remove(ProbeInstr);103 MBB.insert(FirstInstr, ProbeInstr);104 Changed = true;105 }106 } else {107 // Probes not surrounded by any real instructions in the same block are108 // called dangling probes. Since there's no good way to pick up a sample109 // collection point for dangling probes at compile time, they are being110 // removed so that the profile correlation tool will not report any111 // samples collected for them and it's up to the counts inference tool112 // to get them a reasonable count.113 SmallVector<MachineInstr *, 4> ToBeRemoved;114 for (MachineInstr &MI : MBB) {115 if (MI.isPseudoProbe())116 ToBeRemoved.push_back(&MI);117 }118 119 for (auto *MI : ToBeRemoved)120 MI->eraseFromParent();121 122 Changed |= !ToBeRemoved.empty();123 }124 }125 126 return Changed;127 }128 129private:130 uint64_t getFuncGUID(Module *M, DILocation *DL) {131 auto Name = DL->getSubprogramLinkageName();132 return Function::getGUIDAssumingExternalLinkage(Name);133 }134 135 bool ShouldRun = false;136};137} // namespace138 139char PseudoProbeInserter::ID = 0;140INITIALIZE_PASS_BEGIN(PseudoProbeInserter, DEBUG_TYPE,141 "Insert pseudo probe annotations for value profiling",142 false, false)143INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)144INITIALIZE_PASS_END(PseudoProbeInserter, DEBUG_TYPE,145 "Insert pseudo probe annotations for value profiling",146 false, false)147 148FunctionPass *llvm::createPseudoProbeInserter() {149 return new PseudoProbeInserter();150}151