803 lines · cpp
1//===- bolt/Passes/Instrumentation.cpp ------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file implements the Instrumentation class.10//11//===----------------------------------------------------------------------===//12 13#include "bolt/Passes/Instrumentation.h"14#include "bolt/Core/ParallelUtilities.h"15#include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"16#include "bolt/Utils/CommandLineOpts.h"17#include "bolt/Utils/Utils.h"18#include "llvm/Support/CommandLine.h"19#include "llvm/Support/RWMutex.h"20#include <queue>21#include <stack>22#include <unordered_set>23 24#define DEBUG_TYPE "bolt-instrumentation"25 26using namespace llvm;27 28namespace opts {29extern cl::OptionCategory BoltInstrCategory;30 31cl::opt<std::string> InstrumentationFilename(32 "instrumentation-file",33 cl::desc("file name where instrumented profile will be saved (default: "34 "/tmp/prof.fdata)"),35 cl::init("/tmp/prof.fdata"), cl::Optional, cl::cat(BoltInstrCategory));36 37cl::opt<std::string> InstrumentationBinpath(38 "instrumentation-binpath",39 cl::desc("path to instrumented binary in case if /proc/self/map_files "40 "is not accessible due to access restriction issues"),41 cl::Optional, cl::cat(BoltInstrCategory));42 43cl::opt<bool> InstrumentationFileAppendPID(44 "instrumentation-file-append-pid",45 cl::desc("append PID to saved profile file name (default: false)"),46 cl::init(false), cl::Optional, cl::cat(BoltInstrCategory));47 48cl::opt<bool> ConservativeInstrumentation(49 "conservative-instrumentation",50 cl::desc("disable instrumentation optimizations that sacrifice profile "51 "accuracy (for debugging, default: false)"),52 cl::init(false), cl::Optional, cl::cat(BoltInstrCategory));53 54cl::opt<uint32_t> InstrumentationSleepTime(55 "instrumentation-sleep-time",56 cl::desc("interval between profile writes (default: 0 = write only at "57 "program end). This is useful for service workloads when you "58 "want to dump profile every X minutes or if you are killing the "59 "program and the profile is not being dumped at the end."),60 cl::init(0), cl::Optional, cl::cat(BoltInstrCategory));61 62cl::opt<bool> InstrumentationNoCountersClear(63 "instrumentation-no-counters-clear",64 cl::desc("Don't clear counters across dumps "65 "(use with instrumentation-sleep-time option)"),66 cl::init(false), cl::Optional, cl::cat(BoltInstrCategory));67 68cl::opt<bool> InstrumentationWaitForks(69 "instrumentation-wait-forks",70 cl::desc("Wait until all forks of instrumented process will finish "71 "(use with instrumentation-sleep-time option)"),72 cl::init(false), cl::Optional, cl::cat(BoltInstrCategory));73 74cl::opt<bool>75 InstrumentHotOnly("instrument-hot-only",76 cl::desc("only insert instrumentation on hot functions "77 "(needs profile, default: false)"),78 cl::init(false), cl::Optional,79 cl::cat(BoltInstrCategory));80 81cl::opt<bool> InstrumentCalls("instrument-calls",82 cl::desc("record profile for inter-function "83 "control flow activity (default: true)"),84 cl::init(true), cl::Optional,85 cl::cat(BoltInstrCategory));86} // namespace opts87 88namespace llvm {89namespace bolt {90 91static bool hasAArch64ExclusiveMemop(92 BinaryFunction &Function,93 std::unordered_set<const BinaryBasicBlock *> &BBToSkip) {94 // FIXME ARMv8-a architecture reference manual says that software must avoid95 // having any explicit memory accesses between exclusive load and associated96 // store instruction. So for now skip instrumentation for basic blocks that97 // have these instructions, since it might lead to runtime deadlock.98 BinaryContext &BC = Function.getBinaryContext();99 std::queue<std::pair<BinaryBasicBlock *, bool>> BBQueue; // {BB, isLoad}100 std::unordered_set<BinaryBasicBlock *> Visited;101 102 if (Function.getLayout().block_begin() == Function.getLayout().block_end())103 return 0;104 105 BinaryBasicBlock *BBfirst = *Function.getLayout().block_begin();106 BBQueue.push({BBfirst, false});107 108 while (!BBQueue.empty()) {109 BinaryBasicBlock *BB = BBQueue.front().first;110 bool IsLoad = BBQueue.front().second;111 BBQueue.pop();112 if (!Visited.insert(BB).second)113 continue;114 115 for (const MCInst &Inst : *BB) {116 // Two loads one after another - skip whole function117 if (BC.MIB->isAArch64ExclusiveLoad(Inst) && IsLoad) {118 if (opts::Verbosity >= 2) {119 outs() << "BOLT-INSTRUMENTER: function " << Function.getPrintName()120 << " has two exclusive loads. Ignoring the function.\n";121 }122 return true;123 }124 125 if (BC.MIB->isAArch64ExclusiveLoad(Inst))126 IsLoad = true;127 128 if (IsLoad && BBToSkip.insert(BB).second) {129 if (opts::Verbosity >= 2) {130 outs() << "BOLT-INSTRUMENTER: skip BB " << BB->getName()131 << " due to exclusive instruction in function "132 << Function.getPrintName() << "\n";133 }134 }135 136 if (!IsLoad && BC.MIB->isAArch64ExclusiveStore(Inst)) {137 if (opts::Verbosity >= 2) {138 outs() << "BOLT-INSTRUMENTER: function " << Function.getPrintName()139 << " has exclusive store without corresponding load. Ignoring "140 "the function.\n";141 }142 return true;143 }144 145 if (IsLoad && (BC.MIB->isAArch64ExclusiveStore(Inst) ||146 BC.MIB->isAArch64ExclusiveClear(Inst)))147 IsLoad = false;148 }149 150 if (IsLoad && BB->succ_size() == 0) {151 if (opts::Verbosity >= 2) {152 outs()153 << "BOLT-INSTRUMENTER: function " << Function.getPrintName()154 << " has exclusive load in trailing BB. Ignoring the function.\n";155 }156 return true;157 }158 159 for (BinaryBasicBlock *BBS : BB->successors())160 BBQueue.push({BBS, IsLoad});161 }162 163 if (BBToSkip.size() == Visited.size()) {164 if (opts::Verbosity >= 2) {165 outs() << "BOLT-INSTRUMENTER: all BBs are marked with true. Ignoring the "166 "function "167 << Function.getPrintName() << "\n";168 }169 return true;170 }171 172 return false;173}174 175uint32_t Instrumentation::getFunctionNameIndex(const BinaryFunction &Function) {176 auto Iter = FuncToStringIdx.find(&Function);177 if (Iter != FuncToStringIdx.end())178 return Iter->second;179 size_t Idx = Summary->StringTable.size();180 FuncToStringIdx.emplace(std::make_pair(&Function, Idx));181 Summary->StringTable.append(getEscapedName(Function.getOneName()));182 Summary->StringTable.append(1, '\0');183 return Idx;184}185 186bool Instrumentation::createCallDescription(FunctionDescription &FuncDesc,187 const BinaryFunction &FromFunction,188 uint32_t From, uint32_t FromNodeID,189 const BinaryFunction &ToFunction,190 uint32_t To, bool IsInvoke) {191 CallDescription CD;192 // Ordinarily, we don't augment direct calls with an explicit counter, except193 // when forced to do so or when we know this callee could be throwing194 // exceptions, in which case there is no other way to accurately record its195 // frequency.196 bool ForceInstrumentation = opts::ConservativeInstrumentation || IsInvoke;197 CD.FromLoc.FuncString = getFunctionNameIndex(FromFunction);198 CD.FromLoc.Offset = From;199 CD.FromNode = FromNodeID;200 CD.Target = &ToFunction;201 CD.ToLoc.FuncString = getFunctionNameIndex(ToFunction);202 CD.ToLoc.Offset = To;203 CD.Counter = ForceInstrumentation ? Summary->Counters.size() : 0xffffffff;204 if (ForceInstrumentation)205 ++DirectCallCounters;206 FuncDesc.Calls.emplace_back(CD);207 return ForceInstrumentation;208}209 210void Instrumentation::createIndCallDescription(211 const BinaryFunction &FromFunction, uint32_t From) {212 IndCallDescription ICD;213 ICD.FromLoc.FuncString = getFunctionNameIndex(FromFunction);214 ICD.FromLoc.Offset = From;215 Summary->IndCallDescriptions.emplace_back(ICD);216}217 218void Instrumentation::createIndCallTargetDescription(219 const BinaryFunction &ToFunction, uint32_t To) {220 IndCallTargetDescription ICD;221 ICD.ToLoc.FuncString = getFunctionNameIndex(ToFunction);222 ICD.ToLoc.Offset = To;223 ICD.Target = &ToFunction;224 Summary->IndCallTargetDescriptions.emplace_back(ICD);225}226 227bool Instrumentation::createEdgeDescription(FunctionDescription &FuncDesc,228 const BinaryFunction &FromFunction,229 uint32_t From, uint32_t FromNodeID,230 const BinaryFunction &ToFunction,231 uint32_t To, uint32_t ToNodeID,232 bool Instrumented) {233 EdgeDescription ED;234 auto Result = FuncDesc.EdgesSet.insert(std::make_pair(FromNodeID, ToNodeID));235 // Avoid creating duplicated edge descriptions. This happens in CFGs where a236 // block jumps to its fall-through.237 if (Result.second == false)238 return false;239 ED.FromLoc.FuncString = getFunctionNameIndex(FromFunction);240 ED.FromLoc.Offset = From;241 ED.FromNode = FromNodeID;242 ED.ToLoc.FuncString = getFunctionNameIndex(ToFunction);243 ED.ToLoc.Offset = To;244 ED.ToNode = ToNodeID;245 ED.Counter = Instrumented ? Summary->Counters.size() : 0xffffffff;246 if (Instrumented)247 ++BranchCounters;248 FuncDesc.Edges.emplace_back(ED);249 return Instrumented;250}251 252void Instrumentation::createLeafNodeDescription(FunctionDescription &FuncDesc,253 uint32_t Node) {254 InstrumentedNode IN;255 IN.Node = Node;256 IN.Counter = Summary->Counters.size();257 ++LeafNodeCounters;258 FuncDesc.LeafNodes.emplace_back(IN);259}260 261InstructionListType262Instrumentation::createInstrumentationSnippet(BinaryContext &BC, bool IsLeaf) {263 auto L = BC.scopeLock();264 MCSymbol *Label = BC.Ctx->createNamedTempSymbol("InstrEntry");265 Summary->Counters.emplace_back(Label);266 return BC.MIB->createInstrIncMemory(Label, BC.Ctx.get(), IsLeaf,267 BC.AsmInfo->getCodePointerSize());268}269 270// Helper instruction sequence insertion function271static BinaryBasicBlock::iterator272insertInstructions(InstructionListType &Instrs, BinaryBasicBlock &BB,273 BinaryBasicBlock::iterator Iter) {274 for (MCInst &NewInst : Instrs) {275 Iter = BB.insertInstruction(Iter, NewInst);276 ++Iter;277 }278 return Iter;279}280 281void Instrumentation::instrumentLeafNode(BinaryBasicBlock &BB,282 BinaryBasicBlock::iterator Iter,283 bool IsLeaf,284 FunctionDescription &FuncDesc,285 uint32_t Node) {286 createLeafNodeDescription(FuncDesc, Node);287 InstructionListType CounterInstrs = createInstrumentationSnippet(288 BB.getFunction()->getBinaryContext(), IsLeaf);289 insertInstructions(CounterInstrs, BB, Iter);290}291 292void Instrumentation::instrumentIndirectTarget(BinaryBasicBlock &BB,293 BinaryBasicBlock::iterator &Iter,294 BinaryFunction &FromFunction,295 uint32_t From) {296 auto L = FromFunction.getBinaryContext().scopeLock();297 const size_t IndCallSiteID = Summary->IndCallDescriptions.size();298 createIndCallDescription(FromFunction, From);299 300 BinaryContext &BC = FromFunction.getBinaryContext();301 bool IsTailCall = BC.MIB->isTailCall(*Iter);302 InstructionListType CounterInstrs = BC.MIB->createInstrumentedIndirectCall(303 std::move(*Iter),304 IsTailCall ? IndTailCallHandlerExitBBFunction->getSymbol()305 : IndCallHandlerExitBBFunction->getSymbol(),306 IndCallSiteID, &*BC.Ctx);307 308 if (!BC.isAArch64()) {309 Iter = BB.eraseInstruction(Iter);310 Iter = insertInstructions(CounterInstrs, BB, Iter);311 --Iter;312 } else313 Iter = insertInstructions(CounterInstrs, BB, Iter);314}315 316bool Instrumentation::instrumentOneTarget(317 SplitWorklistTy &SplitWorklist, SplitInstrsTy &SplitInstrs,318 BinaryBasicBlock::iterator &Iter, BinaryFunction &FromFunction,319 BinaryBasicBlock &FromBB, uint32_t From, BinaryFunction &ToFunc,320 BinaryBasicBlock *TargetBB, uint32_t ToOffset, bool IsLeaf, bool IsInvoke,321 FunctionDescription *FuncDesc, uint32_t FromNodeID, uint32_t ToNodeID) {322 BinaryContext &BC = FromFunction.getBinaryContext();323 {324 auto L = BC.scopeLock();325 bool Created = true;326 if (!TargetBB)327 Created = createCallDescription(*FuncDesc, FromFunction, From, FromNodeID,328 ToFunc, ToOffset, IsInvoke);329 else330 Created = createEdgeDescription(*FuncDesc, FromFunction, From, FromNodeID,331 ToFunc, ToOffset, ToNodeID,332 /*Instrumented=*/true);333 if (!Created)334 return false;335 }336 337 InstructionListType CounterInstrs = createInstrumentationSnippet(BC, IsLeaf);338 339 const MCInst &Inst = *Iter;340 if (BC.MIB->isCall(Inst)) {341 // This code handles both342 // - (regular) inter-function calls (cross-function control transfer),343 // - (rare) intra-function calls (function-local control transfer)344 Iter = insertInstructions(CounterInstrs, FromBB, Iter);345 return true;346 }347 348 if (!TargetBB || !FuncDesc)349 return false;350 351 // Indirect branch, conditional branches or fall-throughs352 // Regular cond branch, put counter at start of target block353 //354 // N.B.: (FromBB != TargetBBs) checks below handle conditional jumps where355 // we can't put the instrumentation counter in this block because not all356 // paths that reach it at this point will be taken and going to the target.357 if (TargetBB->pred_size() == 1 && &FromBB != TargetBB &&358 !TargetBB->isEntryPoint()) {359 insertInstructions(CounterInstrs, *TargetBB, TargetBB->begin());360 return true;361 }362 if (FromBB.succ_size() == 1 && &FromBB != TargetBB) {363 Iter = insertInstructions(CounterInstrs, FromBB, Iter);364 return true;365 }366 // Critical edge, create BB and put counter there367 SplitWorklist.emplace_back(&FromBB, TargetBB);368 SplitInstrs.emplace_back(std::move(CounterInstrs));369 return true;370}371 372void Instrumentation::instrumentFunction(BinaryFunction &Function,373 MCPlusBuilder::AllocatorIdTy AllocId) {374 if (Function.hasUnknownControlFlow())375 return;376 377 BinaryContext &BC = Function.getBinaryContext();378 if (BC.isMachO() && Function.hasName("___GLOBAL_init_65535/1"))379 return;380 381 std::unordered_set<const BinaryBasicBlock *> BBToSkip;382 if (BC.isAArch64() && hasAArch64ExclusiveMemop(Function, BBToSkip))383 return;384 385 SplitWorklistTy SplitWorklist;386 SplitInstrsTy SplitInstrs;387 388 FunctionDescription *FuncDesc = nullptr;389 {390 std::unique_lock<llvm::sys::RWMutex> L(FDMutex);391 Summary->FunctionDescriptions.emplace_back();392 FuncDesc = &Summary->FunctionDescriptions.back();393 }394 395 FuncDesc->Function = &Function;396 Function.disambiguateJumpTables(AllocId);397 Function.deleteConservativeEdges();398 399 std::unordered_map<const BinaryBasicBlock *, uint32_t> BBToID;400 uint32_t Id = 0;401 for (auto BBI = Function.begin(); BBI != Function.end(); ++BBI) {402 BBToID[&*BBI] = Id++;403 }404 std::unordered_set<const BinaryBasicBlock *> VisitedSet;405 // DFS to establish edges we will use for a spanning tree. Edges in the406 // spanning tree can be instrumentation-free since their count can be407 // inferred by solving flow equations on a bottom-up traversal of the tree.408 // Exit basic blocks are always instrumented so we start the traversal with409 // a minimum number of defined variables to make the equation solvable.410 std::stack<std::pair<const BinaryBasicBlock *, BinaryBasicBlock *>> Stack;411 std::unordered_map<const BinaryBasicBlock *,412 std::set<const BinaryBasicBlock *>>413 STOutSet;414 for (auto BBI = Function.getLayout().block_rbegin();415 BBI != Function.getLayout().block_rend(); ++BBI) {416 if ((*BBI)->isEntryPoint() || (*BBI)->isLandingPad()) {417 Stack.push(std::make_pair(nullptr, *BBI));418 if (opts::InstrumentCalls && (*BBI)->isEntryPoint()) {419 EntryNode E;420 E.Node = BBToID[&**BBI];421 E.Address = (*BBI)->getInputOffset();422 FuncDesc->EntryNodes.emplace_back(E);423 createIndCallTargetDescription(Function, (*BBI)->getInputOffset());424 }425 }426 }427 428 // Modified version of BinaryFunction::dfs() to build a spanning tree429 if (!opts::ConservativeInstrumentation) {430 while (!Stack.empty()) {431 BinaryBasicBlock *BB;432 const BinaryBasicBlock *Pred;433 std::tie(Pred, BB) = Stack.top();434 Stack.pop();435 if (llvm::is_contained(VisitedSet, BB))436 continue;437 438 VisitedSet.insert(BB);439 if (Pred)440 STOutSet[Pred].insert(BB);441 442 for (BinaryBasicBlock *SuccBB : BB->successors())443 Stack.push(std::make_pair(BB, SuccBB));444 }445 }446 447 // Determine whether this is a leaf function, which needs special448 // instructions to protect the red zone449 bool IsLeafFunction = true;450 DenseSet<const BinaryBasicBlock *> InvokeBlocks;451 for (const BinaryBasicBlock &BB : Function) {452 for (const MCInst &Inst : BB) {453 if (BC.MIB->isCall(Inst)) {454 if (BC.MIB->isInvoke(Inst))455 InvokeBlocks.insert(&BB);456 if (!BC.MIB->isTailCall(Inst))457 IsLeafFunction = false;458 }459 }460 }461 462 for (auto BBI = Function.begin(), BBE = Function.end(); BBI != BBE; ++BBI) {463 BinaryBasicBlock &BB = *BBI;464 465 // Skip BBs with exclusive load/stores466 if (BBToSkip.find(&BB) != BBToSkip.end())467 continue;468 469 bool HasUnconditionalBranch = false;470 bool HasJumpTable = false;471 bool IsInvokeBlock = InvokeBlocks.count(&BB) > 0;472 473 for (auto I = BB.begin(); I != BB.end(); ++I) {474 const MCInst &Inst = *I;475 if (!BC.MIB->getOffset(Inst))476 continue;477 478 const bool IsJumpTable = Function.getJumpTable(Inst);479 if (IsJumpTable)480 HasJumpTable = true;481 else if (BC.MIB->isUnconditionalBranch(Inst))482 HasUnconditionalBranch = true;483 else if (!(BC.MIB->isCall(Inst) || BC.MIB->isConditionalBranch(Inst)))484 continue;485 486 const uint32_t FromOffset = *BC.MIB->getOffset(Inst);487 const MCSymbol *Target = BC.MIB->getTargetSymbol(Inst);488 BinaryBasicBlock *TargetBB = Function.getBasicBlockForLabel(Target);489 uint32_t ToOffset = TargetBB ? TargetBB->getInputOffset() : 0;490 BinaryFunction *TargetFunc =491 TargetBB ? &Function : BC.getFunctionForSymbol(Target);492 if (TargetFunc && BC.MIB->isCall(Inst)) {493 if (opts::InstrumentCalls) {494 const BinaryBasicBlock *ForeignBB =495 TargetFunc->getBasicBlockForLabel(Target);496 if (ForeignBB)497 ToOffset = ForeignBB->getInputOffset();498 instrumentOneTarget(SplitWorklist, SplitInstrs, I, Function, BB,499 FromOffset, *TargetFunc, TargetBB, ToOffset,500 IsLeafFunction, IsInvokeBlock, FuncDesc,501 BBToID[&BB]);502 }503 continue;504 }505 if (TargetFunc) {506 // Do not instrument edges in the spanning tree507 if (llvm::is_contained(STOutSet[&BB], TargetBB)) {508 auto L = BC.scopeLock();509 createEdgeDescription(*FuncDesc, Function, FromOffset, BBToID[&BB],510 Function, ToOffset, BBToID[TargetBB],511 /*Instrumented=*/false);512 continue;513 }514 instrumentOneTarget(SplitWorklist, SplitInstrs, I, Function, BB,515 FromOffset, *TargetFunc, TargetBB, ToOffset,516 IsLeafFunction, IsInvokeBlock, FuncDesc,517 BBToID[&BB], BBToID[TargetBB]);518 continue;519 }520 521 if (IsJumpTable) {522 for (BinaryBasicBlock *&Succ : BB.successors()) {523 // Do not instrument edges in the spanning tree524 if (llvm::is_contained(STOutSet[&BB], &*Succ)) {525 auto L = BC.scopeLock();526 createEdgeDescription(*FuncDesc, Function, FromOffset, BBToID[&BB],527 Function, Succ->getInputOffset(),528 BBToID[&*Succ], /*Instrumented=*/false);529 continue;530 }531 instrumentOneTarget(532 SplitWorklist, SplitInstrs, I, Function, BB, FromOffset, Function,533 &*Succ, Succ->getInputOffset(), IsLeafFunction, IsInvokeBlock,534 FuncDesc, BBToID[&BB], BBToID[&*Succ]);535 }536 continue;537 }538 539 // Handle indirect calls -- could be direct calls with unknown targets540 // or secondary entry points of known functions, so check it is indirect541 // to be sure.542 if (opts::InstrumentCalls && BC.MIB->isIndirectCall(*I))543 instrumentIndirectTarget(BB, I, Function, FromOffset);544 545 } // End of instructions loop546 547 // Instrument fallthroughs (when the direct jump instruction is missing)548 if (!HasUnconditionalBranch && !HasJumpTable && BB.succ_size() > 0 &&549 BB.size() > 0) {550 BinaryBasicBlock *FTBB = BB.getFallthrough();551 assert(FTBB && "expected valid fall-through basic block");552 auto I = BB.begin();553 auto LastInstr = BB.end();554 --LastInstr;555 while (LastInstr != I && BC.MIB->isPseudo(*LastInstr))556 --LastInstr;557 uint32_t FromOffset = 0;558 // The last instruction in the BB should have an annotation, except559 // if it was branching to the end of the function as a result of560 // __builtin_unreachable(), in which case it was deleted by fixBranches.561 // Ignore this case. FIXME: force fixBranches() to preserve the offset.562 if (!BC.MIB->getOffset(*LastInstr))563 continue;564 FromOffset = *BC.MIB->getOffset(*LastInstr);565 566 // Do not instrument edges in the spanning tree567 if (llvm::is_contained(STOutSet[&BB], FTBB)) {568 auto L = BC.scopeLock();569 createEdgeDescription(*FuncDesc, Function, FromOffset, BBToID[&BB],570 Function, FTBB->getInputOffset(), BBToID[FTBB],571 /*Instrumented=*/false);572 continue;573 }574 instrumentOneTarget(SplitWorklist, SplitInstrs, I, Function, BB,575 FromOffset, Function, FTBB, FTBB->getInputOffset(),576 IsLeafFunction, IsInvokeBlock, FuncDesc, BBToID[&BB],577 BBToID[FTBB]);578 }579 } // End of BBs loop580 581 // Instrument spanning tree leaves582 if (!opts::ConservativeInstrumentation) {583 for (auto BBI = Function.begin(), BBE = Function.end(); BBI != BBE; ++BBI) {584 BinaryBasicBlock &BB = *BBI;585 if (STOutSet[&BB].size() == 0)586 instrumentLeafNode(BB, BB.begin(), IsLeafFunction, *FuncDesc,587 BBToID[&BB]);588 }589 }590 591 // Consume list of critical edges: split them and add instrumentation to the592 // newly created BBs593 auto Iter = SplitInstrs.begin();594 for (std::pair<BinaryBasicBlock *, BinaryBasicBlock *> &BBPair :595 SplitWorklist) {596 BinaryBasicBlock *NewBB = Function.splitEdge(BBPair.first, BBPair.second);597 NewBB->addInstructions(Iter->begin(), Iter->end());598 ++Iter;599 }600 601 // Unused now602 FuncDesc->EdgesSet.clear();603}604 605Error Instrumentation::runOnFunctions(BinaryContext &BC) {606 const unsigned Flags = BinarySection::getFlags(/*IsReadOnly=*/false,607 /*IsText=*/false,608 /*IsAllocatable=*/true);609 BC.registerOrUpdateSection(".bolt.instr.counters", ELF::SHT_PROGBITS, Flags,610 nullptr, 0, BC.RegularPageSize);611 612 BC.registerOrUpdateNoteSection(".bolt.instr.tables", nullptr, 0,613 /*Alignment=*/1,614 /*IsReadOnly=*/true, ELF::SHT_NOTE);615 616 Summary->IndCallCounterFuncPtr =617 BC.Ctx->getOrCreateSymbol("__bolt_ind_call_counter_func_pointer");618 Summary->IndTailCallCounterFuncPtr =619 BC.Ctx->getOrCreateSymbol("__bolt_ind_tailcall_counter_func_pointer");620 621 createAuxiliaryFunctions(BC);622 623 ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) {624 return (!BF.isSimple() || BF.isIgnored() ||625 (opts::InstrumentHotOnly && !BF.getKnownExecutionCount()));626 };627 628 ParallelUtilities::WorkFuncWithAllocTy WorkFun =629 [&](BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocatorId) {630 instrumentFunction(BF, AllocatorId);631 };632 633 ParallelUtilities::runOnEachFunctionWithUniqueAllocId(634 BC, ParallelUtilities::SchedulingPolicy::SP_INST_QUADRATIC, WorkFun,635 SkipPredicate, "instrumentation", /* ForceSequential=*/true);636 637 if (BC.isMachO()) {638 if (BC.StartFunctionAddress) {639 BinaryFunction *Main =640 BC.getBinaryFunctionAtAddress(*BC.StartFunctionAddress);641 assert(Main && "Entry point function not found");642 BinaryBasicBlock &BB = Main->front();643 644 ErrorOr<BinarySection &> SetupSection =645 BC.getUniqueSectionByName("I__setup");646 if (!SetupSection)647 return createFatalBOLTError("Cannot find I__setup section\n");648 649 MCSymbol *Target = BC.registerNameAtAddress(650 "__bolt_instr_setup", SetupSection->getAddress(), 0, 0);651 MCInst NewInst;652 BC.MIB->createCall(NewInst, Target, BC.Ctx.get());653 BB.insertInstruction(BB.begin(), std::move(NewInst));654 } else {655 BC.errs() << "BOLT-WARNING: Entry point not found\n";656 }657 658 if (BinaryData *BD = BC.getBinaryDataByName("___GLOBAL_init_65535/1")) {659 BinaryFunction *Ctor = BC.getBinaryFunctionAtAddress(BD->getAddress());660 assert(Ctor && "___GLOBAL_init_65535 function not found");661 BinaryBasicBlock &BB = Ctor->front();662 ErrorOr<BinarySection &> FiniSection =663 BC.getUniqueSectionByName("I__fini");664 if (!FiniSection)665 return createFatalBOLTError("Cannot find I__fini section");666 667 MCSymbol *Target = BC.registerNameAtAddress(668 "__bolt_instr_fini", FiniSection->getAddress(), 0, 0);669 auto IsLEA = [&BC](const MCInst &Inst) { return BC.MIB->isLEA64r(Inst); };670 const auto LEA = std::find_if(671 std::next(llvm::find_if(reverse(BB), IsLEA)), BB.rend(), IsLEA);672 LEA->getOperand(4).setExpr(MCSymbolRefExpr::create(Target, *BC.Ctx));673 } else {674 BC.errs() << "BOLT-WARNING: ___GLOBAL_init_65535 not found\n";675 }676 }677 678 setupRuntimeLibrary(BC);679 return Error::success();680}681 682void Instrumentation::createAuxiliaryFunctions(BinaryContext &BC) {683 auto createSimpleFunction =684 [&](StringRef Title, InstructionListType Instrs) -> BinaryFunction * {685 BinaryFunction *Func = BC.createInjectedBinaryFunction(std::string(Title));686 687 std::vector<std::unique_ptr<BinaryBasicBlock>> BBs;688 BBs.emplace_back(Func->createBasicBlock());689 BBs.back()->addInstructions(Instrs.begin(), Instrs.end());690 BBs.back()->setCFIState(0);691 Func->insertBasicBlocks(nullptr, std::move(BBs),692 /*UpdateLayout=*/true,693 /*UpdateCFIState=*/false);694 Func->updateState(BinaryFunction::State::CFG_Finalized);695 return Func;696 };697 698 // Here we are creating a set of functions to handle BB entry/exit.699 // IndCallHandlerExitBB contains instructions to finish handling traffic to an700 // indirect call. We pass it to createInstrumentedIndCallHandlerEntryBB(),701 // which will check if a pointer to runtime library traffic accounting702 // function was initialized (it is done during initialization of runtime703 // library). If it is so - calls it. Then this routine returns to normal704 // execution by jumping to exit BB.705 BinaryFunction *IndCallHandlerExitBB =706 createSimpleFunction("__bolt_instr_ind_call_handler",707 BC.MIB->createInstrumentedIndCallHandlerExitBB());708 709 IndCallHandlerExitBBFunction =710 createSimpleFunction("__bolt_instr_ind_call_handler_func",711 BC.MIB->createInstrumentedIndCallHandlerEntryBB(712 Summary->IndCallCounterFuncPtr,713 IndCallHandlerExitBB->getSymbol(), &*BC.Ctx));714 715 BinaryFunction *IndTailCallHandlerExitBB = createSimpleFunction(716 "__bolt_instr_ind_tail_call_handler",717 BC.MIB->createInstrumentedIndTailCallHandlerExitBB());718 719 IndTailCallHandlerExitBBFunction = createSimpleFunction(720 "__bolt_instr_ind_tailcall_handler_func",721 BC.MIB->createInstrumentedIndCallHandlerEntryBB(722 Summary->IndTailCallCounterFuncPtr,723 IndTailCallHandlerExitBB->getSymbol(), &*BC.Ctx));724 725 createSimpleFunction("__bolt_num_counters_getter",726 BC.MIB->createNumCountersGetter(BC.Ctx.get()));727 createSimpleFunction("__bolt_instr_locations_getter",728 BC.MIB->createInstrLocationsGetter(BC.Ctx.get()));729 createSimpleFunction("__bolt_instr_tables_getter",730 BC.MIB->createInstrTablesGetter(BC.Ctx.get()));731 createSimpleFunction("__bolt_instr_num_funcs_getter",732 BC.MIB->createInstrNumFuncsGetter(BC.Ctx.get()));733 734 if (BC.isELF()) {735 if (BC.StartFunctionAddress) {736 BinaryFunction *Start =737 BC.getBinaryFunctionAtAddress(*BC.StartFunctionAddress);738 assert(Start && "Entry point function not found");739 const MCSymbol *StartSym = Start->getSymbol();740 createSimpleFunction(741 "__bolt_start_trampoline",742 BC.MIB->createSymbolTrampoline(StartSym, BC.Ctx.get()));743 }744 if (BC.FiniFunctionAddress) {745 BinaryFunction *Fini =746 BC.getBinaryFunctionAtAddress(*BC.FiniFunctionAddress);747 assert(Fini && "Finalization function not found");748 const MCSymbol *FiniSym = Fini->getSymbol();749 createSimpleFunction(750 "__bolt_fini_trampoline",751 BC.MIB->createSymbolTrampoline(FiniSym, BC.Ctx.get()));752 } else {753 // Create dummy return function for trampoline to avoid issues754 // with unknown symbol in runtime library. E.g. for static PIE755 // executable756 createSimpleFunction("__bolt_fini_trampoline",757 BC.MIB->createReturnInstructionList(BC.Ctx.get()));758 }759 if (BC.isAArch64())760 BC.MIB->createInstrCounterIncrFunc(BC);761 }762}763 764void Instrumentation::setupRuntimeLibrary(BinaryContext &BC) {765 uint32_t FuncDescSize = Summary->getFDSize();766 767 BC.outs() << "BOLT-INSTRUMENTER: Number of indirect call site descriptors: "768 << Summary->IndCallDescriptions.size() << "\n";769 BC.outs() << "BOLT-INSTRUMENTER: Number of indirect call target descriptors: "770 << Summary->IndCallTargetDescriptions.size() << "\n";771 BC.outs() << "BOLT-INSTRUMENTER: Number of function descriptors: "772 << Summary->FunctionDescriptions.size() << "\n";773 BC.outs() << "BOLT-INSTRUMENTER: Number of branch counters: "774 << BranchCounters << "\n";775 BC.outs() << "BOLT-INSTRUMENTER: Number of ST leaf node counters: "776 << LeafNodeCounters << "\n";777 BC.outs() << "BOLT-INSTRUMENTER: Number of direct call counters: "778 << DirectCallCounters << "\n";779 BC.outs() << "BOLT-INSTRUMENTER: Total number of counters: "780 << Summary->Counters.size() << "\n";781 BC.outs() << "BOLT-INSTRUMENTER: Total size of counters: "782 << (Summary->Counters.size() * 8)783 << " bytes (static alloc memory)\n";784 BC.outs() << "BOLT-INSTRUMENTER: Total size of string table emitted: "785 << Summary->StringTable.size() << " bytes in file\n";786 BC.outs() << "BOLT-INSTRUMENTER: Total size of descriptors: "787 << (FuncDescSize +788 Summary->IndCallDescriptions.size() *789 sizeof(IndCallDescription) +790 Summary->IndCallTargetDescriptions.size() *791 sizeof(IndCallTargetDescription))792 << " bytes in file\n";793 BC.outs() << "BOLT-INSTRUMENTER: Profile will be saved to file "794 << opts::InstrumentationFilename << "\n";795 796 InstrumentationRuntimeLibrary *RtLibrary =797 static_cast<InstrumentationRuntimeLibrary *>(BC.getRuntimeLibrary());798 assert(RtLibrary && "instrumentation runtime library object must be set");799 RtLibrary->setSummary(std::move(Summary));800}801} // namespace bolt802} // namespace llvm803