1882 lines · cpp
1//===- bolt/runtime/instr.cpp ---------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// BOLT runtime instrumentation library for x86 Linux. Currently, BOLT does10// not support linking modules with dependencies on one another into the final11// binary (TODO?), which means this library has to be self-contained in a single12// module.13//14// All extern declarations here need to be defined by BOLT itself. Those will be15// undefined symbols that BOLT needs to resolve by emitting these symbols with16// MCStreamer. Currently, Passes/Instrumentation.cpp is the pass responsible17// for defining the symbols here and these two files have a tight coupling: one18// working statically when you run BOLT and another during program runtime when19// you run an instrumented binary. The main goal here is to output an fdata file20// (BOLT profile) with the instrumentation counters inserted by the static pass.21// Counters for indirect calls are an exception, as we can't know them22// statically. These counters are created and managed here. To allow this, we23// need a minimal framework for allocating memory dynamically. We provide this24// with the BumpPtrAllocator class (not LLVM's, but our own version of it).25//26// Since this code is intended to be inserted into any executable, we decided to27// make it standalone and do not depend on any external libraries (i.e. language28// support libraries, such as glibc or stdc++). To allow this, we provide a few29// light implementations of common OS interacting functionalities using direct30// syscall wrappers. Our simple allocator doesn't manage deallocations that31// fragment the memory space, so it's stack based. This is the minimal framework32// provided here to allow processing instrumented counters and writing fdata.33//34// In the C++ idiom used here, we never use or rely on constructors or35// destructors for global objects. That's because those need support from the36// linker in initialization/finalization code, and we want to keep our linker37// very simple. Similarly, we don't create any global objects that are zero38// initialized, since those would need to go .bss, which our simple linker also39// don't support (TODO?).40//41//===----------------------------------------------------------------------===//42 43#include "common.h"44 45// Enables a very verbose logging to stderr useful when debugging46//#define ENABLE_DEBUG47 48#ifdef ENABLE_DEBUG49#define DEBUG(X) \50 { X; }51#else52#define DEBUG(X) \53 {}54#endif55 56#pragma GCC visibility push(hidden)57 58extern "C" {59 60#if defined(__APPLE__)61extern uint64_t* _bolt_instr_locations_getter();62extern uint32_t _bolt_num_counters_getter();63 64extern uint8_t* _bolt_instr_tables_getter();65extern uint32_t _bolt_instr_num_funcs_getter();66 67#else68 69// Main counters inserted by instrumentation, incremented during runtime when70// points of interest (locations) in the program are reached. Those are direct71// calls and direct and indirect branches (local ones). There are also counters72// for basic block execution if they are a spanning tree leaf and need to be73// counted in order to infer the execution count of other edges of the CFG.74extern uint64_t __bolt_instr_locations[];75extern uint32_t __bolt_num_counters;76// Descriptions are serialized metadata about binary functions written by BOLT,77// so we have a minimal understanding about the program structure. For a78// reference on the exact format of this metadata, see *Description structs,79// Location, IntrumentedNode and EntryNode.80// Number of indirect call site descriptions81extern uint32_t __bolt_instr_num_ind_calls;82// Number of indirect call target descriptions83extern uint32_t __bolt_instr_num_ind_targets;84// Number of function descriptions85extern uint32_t __bolt_instr_num_funcs;86// Time to sleep across dumps (when we write the fdata profile to disk)87extern uint32_t __bolt_instr_sleep_time;88// Do not clear counters across dumps, rewrite file with the updated values89extern bool __bolt_instr_no_counters_clear;90// Wait until all forks of instrumented process will finish91extern bool __bolt_instr_wait_forks;92// Filename to dump data to93extern char __bolt_instr_filename[];94// Instumented binary file path95extern char __bolt_instr_binpath[];96// If true, append current PID to the fdata filename when creating it so97// different invocations of the same program can be differentiated.98extern bool __bolt_instr_use_pid;99// Functions that will be used to instrument indirect calls. BOLT static pass100// will identify indirect calls and modify them to load the address in these101// trampolines and call this address instead. BOLT can't use direct calls to102// our handlers because our addresses here are not known at analysis time. We103// only support resolving dependencies from this file to the output of BOLT,104// *not* the other way around.105// TODO: We need better linking support to make that happen.106extern void (*__bolt_ind_call_counter_func_pointer)();107extern void (*__bolt_ind_tailcall_counter_func_pointer)();108// Function pointers to init/fini trampoline routines in the binary, so we can109// resume regular execution of these functions that we hooked110extern void __bolt_start_trampoline();111extern void __bolt_fini_trampoline();112 113#endif114}115 116namespace {117 118/// A simple allocator that mmaps a fixed size region and manages this space119/// in a stack fashion, meaning you always deallocate the last element that120/// was allocated. In practice, we don't need to deallocate individual elements.121/// We monotonically increase our usage and then deallocate everything once we122/// are done processing something.123class BumpPtrAllocator {124 /// This is written before each allocation and act as a canary to detect when125 /// a bug caused our program to cross allocation boundaries.126 struct EntryMetadata {127 uint64_t Magic;128 uint64_t AllocSize;129 };130 131public:132 void *allocate(size_t Size) {133 Lock L(M);134 135 if (StackBase == nullptr) {136 StackBase = reinterpret_cast<uint8_t *>(137 __mmap(0, MaxSize, PROT_READ | PROT_WRITE,138 (Shared ? MAP_SHARED : MAP_PRIVATE) | MAP_ANONYMOUS, -1, 0));139 assert(StackBase != MAP_FAILED,140 "BumpPtrAllocator: failed to mmap stack!");141 StackSize = 0;142 }143 144 Size = alignTo(Size + sizeof(EntryMetadata), 16);145 uint8_t *AllocAddress = StackBase + StackSize + sizeof(EntryMetadata);146 auto *M = reinterpret_cast<EntryMetadata *>(StackBase + StackSize);147 M->Magic = Magic;148 M->AllocSize = Size;149 StackSize += Size;150 assert(StackSize < MaxSize, "allocator ran out of memory");151 return AllocAddress;152 }153 154#ifdef DEBUG155 /// Element-wise deallocation is only used for debugging to catch memory156 /// bugs by checking magic bytes. Ordinarily, we reset the allocator once157 /// we are done with it. Reset is done with clear(). There's no need158 /// to deallocate each element individually.159 void deallocate(void *Ptr) {160 Lock L(M);161 uint8_t MetadataOffset = sizeof(EntryMetadata);162 auto *M = reinterpret_cast<EntryMetadata *>(163 reinterpret_cast<uint8_t *>(Ptr) - MetadataOffset);164 const uint8_t *StackTop = StackBase + StackSize + MetadataOffset;165 // Validate size166 if (Ptr != StackTop - M->AllocSize) {167 // Failed validation, check if it is a pointer returned by operator new []168 MetadataOffset +=169 sizeof(uint64_t); // Space for number of elements alloc'ed170 M = reinterpret_cast<EntryMetadata *>(reinterpret_cast<uint8_t *>(Ptr) -171 MetadataOffset);172 // Ok, it failed both checks if this assertion fails. Stop the program, we173 // have a memory bug.174 assert(Ptr == StackTop - M->AllocSize,175 "must deallocate the last element alloc'ed");176 }177 assert(M->Magic == Magic, "allocator magic is corrupt");178 StackSize -= M->AllocSize;179 }180#else181 void deallocate(void *) {}182#endif183 184 void clear() {185 Lock L(M);186 StackSize = 0;187 }188 189 /// Set mmap reservation size (only relevant before first allocation)190 void setMaxSize(uint64_t Size) { MaxSize = Size; }191 192 /// Set mmap reservation privacy (only relevant before first allocation)193 void setShared(bool S) { Shared = S; }194 195 void destroy() {196 if (StackBase == nullptr)197 return;198 __munmap(StackBase, MaxSize);199 }200 201 // Placement operator to construct allocator in possibly shared mmaped memory202 static void *operator new(size_t, void *Ptr) { return Ptr; };203 204private:205 static constexpr uint64_t Magic = 0x1122334455667788ull;206 uint64_t MaxSize = 0xa00000;207 uint8_t *StackBase{nullptr};208 uint64_t StackSize{0};209 bool Shared{false};210 Mutex M;211};212 213/// Used for allocating indirect call instrumentation counters. Initialized by214/// __bolt_instr_setup, our initialization routine.215BumpPtrAllocator *GlobalAlloc;216 217// Base address which we subtract from recorded PC values when searching for218// indirect call description entries. Needed because indCall descriptions are219// mapped read-only and contain static addresses. Initialized in220// __bolt_instr_setup.221uint64_t TextBaseAddress = 0;222 223// Storage for GlobalAlloc which can be shared if not using224// instrumentation-file-append-pid.225void *GlobalMetadataStorage;226 227} // anonymous namespace228 229// User-defined placement new operators. We only use those (as opposed to230// overriding the regular operator new) so we can keep our allocator in the231// stack instead of in a data section (global).232void *operator new(size_t Sz, BumpPtrAllocator &A) { return A.allocate(Sz); }233void *operator new(size_t Sz, BumpPtrAllocator &A, char C) {234 auto *Ptr = reinterpret_cast<char *>(A.allocate(Sz));235 memset(Ptr, C, Sz);236 return Ptr;237}238void *operator new[](size_t Sz, BumpPtrAllocator &A) {239 return A.allocate(Sz);240}241void *operator new[](size_t Sz, BumpPtrAllocator &A, char C) {242 auto *Ptr = reinterpret_cast<char *>(A.allocate(Sz));243 memset(Ptr, C, Sz);244 return Ptr;245}246// Only called during exception unwinding (useless). We must manually dealloc.247// C++ language weirdness248void operator delete(void *Ptr, BumpPtrAllocator &A) { A.deallocate(Ptr); }249 250namespace {251 252// Disable instrumentation optimizations that sacrifice profile accuracy253extern "C" bool __bolt_instr_conservative;254 255/// Basic key-val atom stored in our hash256struct SimpleHashTableEntryBase {257 uint64_t Key;258 uint64_t Val;259 void dump(const char *Msg = nullptr) {260 // TODO: make some sort of formatting function261 // Currently we have to do it the ugly way because262 // we want every message to be printed atomically via a single call to263 // __write. If we use reportNumber() and others nultiple times, we'll get264 // garbage in multithreaded environment265 char Buf[BufSize];266 char *Ptr = Buf;267 Ptr = intToStr(Ptr, __getpid(), 10);268 *Ptr++ = ':';269 *Ptr++ = ' ';270 if (Msg)271 Ptr = strCopy(Ptr, Msg, strLen(Msg));272 *Ptr++ = '0';273 *Ptr++ = 'x';274 Ptr = intToStr(Ptr, (uint64_t)this, 16);275 *Ptr++ = ':';276 *Ptr++ = ' ';277 Ptr = strCopy(Ptr, "MapEntry(0x", sizeof("MapEntry(0x") - 1);278 Ptr = intToStr(Ptr, Key, 16);279 *Ptr++ = ',';280 *Ptr++ = ' ';281 *Ptr++ = '0';282 *Ptr++ = 'x';283 Ptr = intToStr(Ptr, Val, 16);284 *Ptr++ = ')';285 *Ptr++ = '\n';286 assert(Ptr - Buf < BufSize, "Buffer overflow!");287 // print everything all at once for atomicity288 __write(2, Buf, Ptr - Buf);289 }290};291 292/// This hash table implementation starts by allocating a table of size293/// InitialSize. When conflicts happen in this main table, it resolves294/// them by chaining a new table of size IncSize. It never reallocs as our295/// allocator doesn't support it. The key is intended to be function pointers.296/// There's no clever hash function (it's just x mod size, size being prime).297/// I never tuned the coefficientes in the modular equation (TODO)298/// This is used for indirect calls (each call site has one of this, so it299/// should have a small footprint) and for tallying call counts globally for300/// each target to check if we missed the origin of some calls (this one is a301/// large instantiation of this template, since it is global for all call sites)302template <typename T = SimpleHashTableEntryBase, uint32_t InitialSize = 7,303 uint32_t IncSize = 7>304class SimpleHashTable {305public:306 using MapEntry = T;307 308 /// Increment by 1 the value of \p Key. If it is not in this table, it will be309 /// added to the table and its value set to 1.310 void incrementVal(uint64_t Key, BumpPtrAllocator &Alloc) {311 if (!__bolt_instr_conservative) {312 TryLock L(M);313 if (!L.isLocked())314 return;315 auto &E = getOrAllocEntry(Key, Alloc);316 ++E.Val;317 return;318 }319 Lock L(M);320 auto &E = getOrAllocEntry(Key, Alloc);321 ++E.Val;322 }323 324 /// Basic member accessing interface. Here we pass the allocator explicitly to325 /// avoid storing a pointer to it as part of this table (remember there is one326 /// hash for each indirect call site, so we want to minimize our footprint).327 MapEntry &get(uint64_t Key, BumpPtrAllocator &Alloc) {328 if (!__bolt_instr_conservative) {329 TryLock L(M);330 if (!L.isLocked())331 return NoEntry;332 return getOrAllocEntry(Key, Alloc);333 }334 Lock L(M);335 return getOrAllocEntry(Key, Alloc);336 }337 338 /// Traverses all elements in the table339 template <typename... Args>340 void forEachElement(void (*Callback)(MapEntry &, Args...), Args... args) {341 Lock L(M);342 if (!TableRoot)343 return;344 return forEachElement(Callback, InitialSize, TableRoot, args...);345 }346 347 void resetCounters();348 349private:350 constexpr static uint64_t VacantMarker = 0;351 constexpr static uint64_t FollowUpTableMarker = 0x8000000000000000ull;352 353 MapEntry *TableRoot{nullptr};354 MapEntry NoEntry;355 Mutex M;356 357 template <typename... Args>358 void forEachElement(void (*Callback)(MapEntry &, Args...),359 uint32_t NumEntries, MapEntry *Entries, Args... args) {360 for (uint32_t I = 0; I < NumEntries; ++I) {361 MapEntry &Entry = Entries[I];362 if (Entry.Key == VacantMarker)363 continue;364 if (Entry.Key & FollowUpTableMarker) {365 MapEntry *Next =366 reinterpret_cast<MapEntry *>(Entry.Key & ~FollowUpTableMarker);367 assert(Next != Entries, "Circular reference!");368 forEachElement(Callback, IncSize, Next, args...);369 continue;370 }371 Callback(Entry, args...);372 }373 }374 375 MapEntry &firstAllocation(uint64_t Key, BumpPtrAllocator &Alloc) {376 TableRoot = new (Alloc, 0) MapEntry[InitialSize];377 MapEntry &Entry = TableRoot[Key % InitialSize];378 Entry.Key = Key;379 // DEBUG(Entry.dump("Created root entry: "));380 return Entry;381 }382 383 MapEntry &getEntry(MapEntry *Entries, uint64_t Key, uint64_t Selector,384 BumpPtrAllocator &Alloc, int CurLevel) {385 // DEBUG(reportNumber("getEntry called, level ", CurLevel, 10));386 const uint32_t NumEntries = CurLevel == 0 ? InitialSize : IncSize;387 uint64_t Remainder = Selector / NumEntries;388 Selector = Selector % NumEntries;389 MapEntry &Entry = Entries[Selector];390 391 // A hit392 if (Entry.Key == Key) {393 // DEBUG(Entry.dump("Hit: "));394 return Entry;395 }396 397 // Vacant - add new entry398 if (Entry.Key == VacantMarker) {399 Entry.Key = Key;400 // DEBUG(Entry.dump("Adding new entry: "));401 return Entry;402 }403 404 // Defer to the next level405 if (Entry.Key & FollowUpTableMarker) {406 return getEntry(407 reinterpret_cast<MapEntry *>(Entry.Key & ~FollowUpTableMarker),408 Key, Remainder, Alloc, CurLevel + 1);409 }410 411 // Conflict - create the next level412 // DEBUG(Entry.dump("Creating new level: "));413 414 MapEntry *NextLevelTbl = new (Alloc, 0) MapEntry[IncSize];415 // DEBUG(416 // reportNumber("Newly allocated level: 0x", uint64_t(NextLevelTbl),417 // 16));418 uint64_t CurEntrySelector = Entry.Key / InitialSize;419 for (int I = 0; I < CurLevel; ++I)420 CurEntrySelector /= IncSize;421 CurEntrySelector = CurEntrySelector % IncSize;422 NextLevelTbl[CurEntrySelector] = Entry;423 Entry.Key = reinterpret_cast<uint64_t>(NextLevelTbl) | FollowUpTableMarker;424 assert((NextLevelTbl[CurEntrySelector].Key & ~FollowUpTableMarker) !=425 uint64_t(Entries),426 "circular reference created!\n");427 // DEBUG(NextLevelTbl[CurEntrySelector].dump("New level entry: "));428 // DEBUG(Entry.dump("Updated old entry: "));429 return getEntry(NextLevelTbl, Key, Remainder, Alloc, CurLevel + 1);430 }431 432 MapEntry &getOrAllocEntry(uint64_t Key, BumpPtrAllocator &Alloc) {433 if (TableRoot) {434 MapEntry &E = getEntry(TableRoot, Key, Key, Alloc, 0);435 assert(!(E.Key & FollowUpTableMarker), "Invalid entry!");436 return E;437 }438 return firstAllocation(Key, Alloc);439 }440};441 442template <typename T> void resetIndCallCounter(T &Entry) {443 Entry.Val = 0;444}445 446template <typename T, uint32_t X, uint32_t Y>447void SimpleHashTable<T, X, Y>::resetCounters() {448 forEachElement(resetIndCallCounter);449}450 451/// Represents a hash table mapping a function target address to its counter.452using IndirectCallHashTable = SimpleHashTable<>;453 454/// Initialize with number 1 instead of 0 so we don't go into .bss. This is the455/// global array of all hash tables storing indirect call destinations happening456/// during runtime, one table per call site.457IndirectCallHashTable *GlobalIndCallCounters{458 reinterpret_cast<IndirectCallHashTable *>(1)};459 460/// Don't allow reentrancy in the fdata writing phase - only one thread writes461/// it462Mutex *GlobalWriteProfileMutex{reinterpret_cast<Mutex *>(1)};463 464/// Store number of calls in additional to target address (Key) and frequency465/// as perceived by the basic block counter (Val).466struct CallFlowEntryBase : public SimpleHashTableEntryBase {467 uint64_t Calls;468};469 470using CallFlowHashTableBase = SimpleHashTable<CallFlowEntryBase, 11939, 233>;471 472/// This is a large table indexing all possible call targets (indirect and473/// direct ones). The goal is to find mismatches between number of calls (for474/// those calls we were able to track) and the entry basic block counter of the475/// callee. In most cases, these two should be equal. If not, there are two476/// possible scenarios here:477///478/// * Entry BB has higher frequency than all known calls to this function.479/// In this case, we have dynamic library code or any uninstrumented code480/// calling this function. We will write the profile for these untracked481/// calls as having source "0 [unknown] 0" in the fdata file.482///483/// * Number of known calls is higher than the frequency of entry BB484/// This only happens when there is no counter for the entry BB / callee485/// function is not simple (in BOLT terms). We don't do anything special486/// here and just ignore those (we still report all calls to the non-simple487/// function, though).488///489class CallFlowHashTable : public CallFlowHashTableBase {490public:491 CallFlowHashTable(BumpPtrAllocator &Alloc) : Alloc(Alloc) {}492 493 MapEntry &get(uint64_t Key) { return CallFlowHashTableBase::get(Key, Alloc); }494 495private:496 // Different than the hash table for indirect call targets, we do store the497 // allocator here since there is only one call flow hash and space overhead498 // is negligible.499 BumpPtrAllocator &Alloc;500};501 502///503/// Description metadata emitted by BOLT to describe the program - refer to504/// Passes/Instrumentation.cpp - Instrumentation::emitTablesAsELFNote()505///506struct Location {507 uint32_t FunctionName;508 uint32_t Offset;509};510 511struct CallDescription {512 Location From;513 uint32_t FromNode;514 Location To;515 uint32_t Counter;516 uint64_t TargetAddress;517};518 519using IndCallDescription = Location;520 521struct IndCallTargetDescription {522 Location Loc;523 uint64_t Address;524};525 526struct EdgeDescription {527 Location From;528 uint32_t FromNode;529 Location To;530 uint32_t ToNode;531 uint32_t Counter;532};533 534struct InstrumentedNode {535 uint32_t Node;536 uint32_t Counter;537};538 539struct EntryNode {540 uint64_t Node;541 uint64_t Address;542};543 544struct FunctionDescription {545 uint32_t NumLeafNodes;546 const InstrumentedNode *LeafNodes;547 uint32_t NumEdges;548 const EdgeDescription *Edges;549 uint32_t NumCalls;550 const CallDescription *Calls;551 uint32_t NumEntryNodes;552 const EntryNode *EntryNodes;553 554 /// Constructor will parse the serialized function metadata written by BOLT555 FunctionDescription(const uint8_t *FuncDesc);556 557 uint64_t getSize() const {558 return 16 + NumLeafNodes * sizeof(InstrumentedNode) +559 NumEdges * sizeof(EdgeDescription) +560 NumCalls * sizeof(CallDescription) +561 NumEntryNodes * sizeof(EntryNode);562 }563};564 565/// The context is created when the fdata profile needs to be written to disk566/// and we need to interpret our runtime counters. It contains pointers to the567/// mmaped binary (only the BOLT written metadata section). Deserialization568/// should be straightforward as most data is POD or an array of POD elements.569/// This metadata is used to reconstruct function CFGs.570struct ProfileWriterContext {571 const IndCallDescription *IndCallDescriptions;572 const IndCallTargetDescription *IndCallTargets;573 const uint8_t *FuncDescriptions;574 const char *Strings; // String table with function names used in this binary575 int FileDesc; // File descriptor for the file on disk backing this576 // information in memory via mmap577 const void *MMapPtr; // The mmap ptr578 int MMapSize; // The mmap size579 580 /// Hash table storing all possible call destinations to detect untracked581 /// calls and correctly report them as [unknown] in output fdata.582 CallFlowHashTable *CallFlowTable;583 584 /// Lookup the sorted indirect call target vector to fetch function name and585 /// offset for an arbitrary function pointer.586 const IndCallTargetDescription *lookupIndCallTarget(uint64_t Target) const;587};588 589/// Perform a string comparison and returns zero if Str1 matches Str2. Compares590/// at most Size characters.591int compareStr(const char *Str1, const char *Str2, int Size) {592 while (*Str1 == *Str2) {593 if (*Str1 == '\0' || --Size == 0)594 return 0;595 ++Str1;596 ++Str2;597 }598 return 1;599}600 601/// Output Location to the fdata file602char *serializeLoc(const ProfileWriterContext &Ctx, char *OutBuf,603 const Location Loc, uint32_t BufSize) {604 // fdata location format: Type Name Offset605 // Type 1 - regular symbol606 OutBuf = strCopy(OutBuf, "1 ");607 const char *Str = Ctx.Strings + Loc.FunctionName;608 uint32_t Size = 25;609 while (*Str) {610 *OutBuf++ = *Str++;611 if (++Size >= BufSize)612 break;613 }614 assert(!*Str, "buffer overflow, function name too large");615 *OutBuf++ = ' ';616 OutBuf = intToStr(OutBuf, Loc.Offset, 16);617 *OutBuf++ = ' ';618 return OutBuf;619}620 621/// Read and deserialize a function description written by BOLT. \p FuncDesc622/// points at the beginning of the function metadata structure in the file.623/// See Instrumentation::emitTablesAsELFNote()624FunctionDescription::FunctionDescription(const uint8_t *FuncDesc) {625 NumLeafNodes = *reinterpret_cast<const uint32_t *>(FuncDesc);626 DEBUG(reportNumber("NumLeafNodes = ", NumLeafNodes, 10));627 LeafNodes = reinterpret_cast<const InstrumentedNode *>(FuncDesc + 4);628 629 NumEdges = *reinterpret_cast<const uint32_t *>(630 FuncDesc + 4 + NumLeafNodes * sizeof(InstrumentedNode));631 DEBUG(reportNumber("NumEdges = ", NumEdges, 10));632 Edges = reinterpret_cast<const EdgeDescription *>(633 FuncDesc + 8 + NumLeafNodes * sizeof(InstrumentedNode));634 635 NumCalls = *reinterpret_cast<const uint32_t *>(636 FuncDesc + 8 + NumLeafNodes * sizeof(InstrumentedNode) +637 NumEdges * sizeof(EdgeDescription));638 DEBUG(reportNumber("NumCalls = ", NumCalls, 10));639 Calls = reinterpret_cast<const CallDescription *>(640 FuncDesc + 12 + NumLeafNodes * sizeof(InstrumentedNode) +641 NumEdges * sizeof(EdgeDescription));642 NumEntryNodes = *reinterpret_cast<const uint32_t *>(643 FuncDesc + 12 + NumLeafNodes * sizeof(InstrumentedNode) +644 NumEdges * sizeof(EdgeDescription) + NumCalls * sizeof(CallDescription));645 DEBUG(reportNumber("NumEntryNodes = ", NumEntryNodes, 10));646 EntryNodes = reinterpret_cast<const EntryNode *>(647 FuncDesc + 16 + NumLeafNodes * sizeof(InstrumentedNode) +648 NumEdges * sizeof(EdgeDescription) + NumCalls * sizeof(CallDescription));649}650 651/// Read and mmap descriptions written by BOLT from the executable's notes652/// section653#if defined(HAVE_ELF_H) and !defined(__APPLE__)654 655void *__attribute__((noinline)) __get_pc() {656 return __builtin_extract_return_addr(__builtin_return_address(0));657}658 659/// Get string with address and parse it to hex pair <StartAddress, EndAddress>660bool parseAddressRange(const char *Str, uint64_t &StartAddress,661 uint64_t &EndAddress) {662 if (!Str)663 return false;664 // Parsed string format: <hex1>-<hex2>665 StartAddress = hexToLong(Str, '-');666 while (*Str && *Str != '-')667 ++Str;668 if (!*Str)669 return false;670 ++Str; // swallow '-'671 EndAddress = hexToLong(Str);672 return true;673}674 675static constexpr uint32_t NameMax = 4096;676static char TargetPath[NameMax] = {};677 678/// Get full path to the real binary by getting current virtual address679/// and searching for the appropriate link in address range in680/// /proc/self/map_files681static char *getBinaryPath() {682 const uint32_t BufSize = 1024;683 const char DirPath[] = "/proc/self/map_files/";684 char Buf[BufSize];685 686 if (__bolt_instr_binpath[0] != '\0')687 return __bolt_instr_binpath;688 689 if (TargetPath[0] != '\0')690 return TargetPath;691 692 unsigned long CurAddr = (unsigned long)__get_pc();693 uint64_t FDdir = __open(DirPath, O_RDONLY,694 /*mode=*/0666);695 assert(static_cast<int64_t>(FDdir) >= 0,696 "failed to open /proc/self/map_files");697 698 while (long Nread = __getdents64(FDdir, (struct dirent64 *)Buf, BufSize)) {699 assert(static_cast<int64_t>(Nread) != -1, "failed to get folder entries");700 701 struct dirent64 *d;702 for (long Bpos = 0; Bpos < Nread; Bpos += d->d_reclen) {703 d = (struct dirent64 *)(Buf + Bpos);704 705 uint64_t StartAddress, EndAddress;706 if (!parseAddressRange(d->d_name, StartAddress, EndAddress))707 continue;708 if (CurAddr < StartAddress || CurAddr > EndAddress)709 continue;710 char FindBuf[NameMax];711 char *C = strCopy(FindBuf, DirPath, NameMax);712 C = strCopy(C, d->d_name, NameMax - (C - FindBuf));713 *C = '\0';714 uint32_t Ret = __readlink(FindBuf, TargetPath, sizeof(TargetPath));715 assert(Ret != -1 && Ret != BufSize, "readlink error");716 TargetPath[Ret] = '\0';717 __close(FDdir);718 return TargetPath;719 }720 }721 __close(FDdir);722 return nullptr;723}724 725ProfileWriterContext readDescriptions(const uint8_t *BinContents,726 uint64_t Size) {727 ProfileWriterContext Result;728 729 assert((BinContents == nullptr) == (Size == 0),730 "either empty or valid library content buffer");731 732 if (BinContents) {733 Result.FileDesc = -1;734 } else {735 const char *BinPath = getBinaryPath();736 assert(BinPath && BinPath[0] != '\0', "failed to find binary path");737 738 uint64_t FD = __open(BinPath, O_RDONLY,739 /*mode=*/0666);740 assert(static_cast<int64_t>(FD) >= 0, "failed to open binary path");741 742 Result.FileDesc = FD;743 744 // mmap our binary to memory745 Size = __lseek(FD, 0, SEEK_END);746 BinContents = reinterpret_cast<uint8_t *>(747 __mmap(0, Size, PROT_READ, MAP_PRIVATE, FD, 0));748 assert(BinContents != MAP_FAILED, "readDescriptions: Failed to mmap self!");749 }750 Result.MMapPtr = BinContents;751 Result.MMapSize = Size;752 const Elf64_Ehdr *Hdr = reinterpret_cast<const Elf64_Ehdr *>(BinContents);753 const Elf64_Shdr *Shdr =754 reinterpret_cast<const Elf64_Shdr *>(BinContents + Hdr->e_shoff);755 const Elf64_Shdr *StringTblHeader = reinterpret_cast<const Elf64_Shdr *>(756 BinContents + Hdr->e_shoff + Hdr->e_shstrndx * Hdr->e_shentsize);757 758 // Find .bolt.instr.tables with the data we need and set pointers to it759 for (int I = 0; I < Hdr->e_shnum; ++I) {760 const char *SecName = reinterpret_cast<const char *>(761 BinContents + StringTblHeader->sh_offset + Shdr->sh_name);762 if (compareStr(SecName, ".bolt.instr.tables", 64) != 0) {763 Shdr = reinterpret_cast<const Elf64_Shdr *>(BinContents + Hdr->e_shoff +764 (I + 1) * Hdr->e_shentsize);765 continue;766 }767 // Actual contents of the ELF note start after offset 20 decimal:768 // Offset 0: Producer name size (4 bytes)769 // Offset 4: Contents size (4 bytes)770 // Offset 8: Note type (4 bytes)771 // Offset 12: Producer name (BOLT\0) (5 bytes + align to 4-byte boundary)772 // Offset 20: Contents773 uint32_t IndCallDescSize =774 *reinterpret_cast<const uint32_t *>(BinContents + Shdr->sh_offset + 20);775 uint32_t IndCallTargetDescSize = *reinterpret_cast<const uint32_t *>(776 BinContents + Shdr->sh_offset + 24 + IndCallDescSize);777 uint32_t FuncDescSize = *reinterpret_cast<const uint32_t *>(778 BinContents + Shdr->sh_offset + 28 + IndCallDescSize +779 IndCallTargetDescSize);780 Result.IndCallDescriptions = reinterpret_cast<const IndCallDescription *>(781 BinContents + Shdr->sh_offset + 24);782 Result.IndCallTargets = reinterpret_cast<const IndCallTargetDescription *>(783 BinContents + Shdr->sh_offset + 28 + IndCallDescSize);784 Result.FuncDescriptions = BinContents + Shdr->sh_offset + 32 +785 IndCallDescSize + IndCallTargetDescSize;786 Result.Strings = reinterpret_cast<const char *>(787 BinContents + Shdr->sh_offset + 32 + IndCallDescSize +788 IndCallTargetDescSize + FuncDescSize);789 return Result;790 }791 const char ErrMsg[] =792 "BOLT instrumentation runtime error: could not find section "793 ".bolt.instr.tables\n";794 reportError(ErrMsg, sizeof(ErrMsg));795 return Result;796}797 798#else799 800ProfileWriterContext readDescriptions() {801 ProfileWriterContext Result;802 uint8_t *Tables = _bolt_instr_tables_getter();803 uint32_t IndCallDescSize = *reinterpret_cast<uint32_t *>(Tables);804 uint32_t IndCallTargetDescSize =805 *reinterpret_cast<uint32_t *>(Tables + 4 + IndCallDescSize);806 uint32_t FuncDescSize = *reinterpret_cast<uint32_t *>(807 Tables + 8 + IndCallDescSize + IndCallTargetDescSize);808 Result.IndCallDescriptions =809 reinterpret_cast<IndCallDescription *>(Tables + 4);810 Result.IndCallTargets = reinterpret_cast<IndCallTargetDescription *>(811 Tables + 8 + IndCallDescSize);812 Result.FuncDescriptions =813 Tables + 12 + IndCallDescSize + IndCallTargetDescSize;814 Result.Strings = reinterpret_cast<char *>(815 Tables + 12 + IndCallDescSize + IndCallTargetDescSize + FuncDescSize);816 return Result;817}818 819#endif820 821#if !defined(__APPLE__)822/// Debug by printing overall metadata global numbers to check it is sane823void printStats(const ProfileWriterContext &Ctx) {824 char StatMsg[BufSize];825 char *StatPtr = StatMsg;826 StatPtr =827 strCopy(StatPtr,828 "\nBOLT INSTRUMENTATION RUNTIME STATISTICS\n\nIndCallDescSize: ");829 StatPtr = intToStr(StatPtr,830 Ctx.FuncDescriptions - reinterpret_cast<const uint8_t *>(831 Ctx.IndCallDescriptions),832 10);833 StatPtr = strCopy(StatPtr, "\nFuncDescSize: ");834 StatPtr = intToStr(StatPtr,835 reinterpret_cast<const uint8_t *>(Ctx.Strings) -836 Ctx.FuncDescriptions,837 10);838 StatPtr = strCopy(StatPtr, "\n__bolt_instr_num_ind_calls: ");839 StatPtr = intToStr(StatPtr, __bolt_instr_num_ind_calls, 10);840 StatPtr = strCopy(StatPtr, "\n__bolt_instr_num_funcs: ");841 StatPtr = intToStr(StatPtr, __bolt_instr_num_funcs, 10);842 StatPtr = strCopy(StatPtr, "\n");843 __write(2, StatMsg, StatPtr - StatMsg);844}845#endif846 847 848/// This is part of a simple CFG representation in memory, where we store849/// a dynamically sized array of input and output edges per node, and store850/// a dynamically sized array of nodes per graph. We also store the spanning851/// tree edges for that CFG in a separate array of nodes in852/// \p SpanningTreeNodes, while the regular nodes live in \p CFGNodes.853struct Edge {854 uint32_t Node; // Index in nodes array regarding the destination of this edge855 uint32_t ID; // Edge index in an array comprising all edges of the graph856};857 858/// A regular graph node or a spanning tree node859struct Node {860 uint32_t NumInEdges{0}; // Input edge count used to size InEdge861 uint32_t NumOutEdges{0}; // Output edge count used to size OutEdges862 Edge *InEdges{nullptr}; // Created and managed by \p Graph863 Edge *OutEdges{nullptr}; // ditto864};865 866/// Main class for CFG representation in memory. Manages object creation and867/// destruction, populates an array of CFG nodes as well as corresponding868/// spanning tree nodes.869struct Graph {870 uint32_t NumNodes;871 Node *CFGNodes;872 Node *SpanningTreeNodes;873 uint64_t *EdgeFreqs;874 uint64_t *CallFreqs;875 BumpPtrAllocator &Alloc;876 const FunctionDescription &D;877 878 /// Reads a list of edges from function description \p D and builds879 /// the graph from it. Allocates several internal dynamic structures that are880 /// later destroyed by ~Graph() and uses \p Alloc. D.LeafNodes contain all881 /// spanning tree leaf nodes descriptions (their counters). They are the seed882 /// used to compute the rest of the missing edge counts in a bottom-up883 /// traversal of the spanning tree.884 Graph(BumpPtrAllocator &Alloc, const FunctionDescription &D,885 const uint64_t *Counters, ProfileWriterContext &Ctx);886 ~Graph();887 void dump() const;888 889private:890 void computeEdgeFrequencies(const uint64_t *Counters,891 ProfileWriterContext &Ctx);892 void dumpEdgeFreqs() const;893};894 895Graph::Graph(BumpPtrAllocator &Alloc, const FunctionDescription &D,896 const uint64_t *Counters, ProfileWriterContext &Ctx)897 : Alloc(Alloc), D(D) {898 DEBUG(reportNumber("G = 0x", (uint64_t)this, 16));899 // First pass to determine number of nodes900 int32_t MaxNodes = -1;901 CallFreqs = nullptr;902 EdgeFreqs = nullptr;903 for (int I = 0; I < D.NumEdges; ++I) {904 if (static_cast<int32_t>(D.Edges[I].FromNode) > MaxNodes)905 MaxNodes = D.Edges[I].FromNode;906 if (static_cast<int32_t>(D.Edges[I].ToNode) > MaxNodes)907 MaxNodes = D.Edges[I].ToNode;908 }909 910 for (int I = 0; I < D.NumLeafNodes; ++I)911 if (static_cast<int32_t>(D.LeafNodes[I].Node) > MaxNodes)912 MaxNodes = D.LeafNodes[I].Node;913 914 for (int I = 0; I < D.NumCalls; ++I)915 if (static_cast<int32_t>(D.Calls[I].FromNode) > MaxNodes)916 MaxNodes = D.Calls[I].FromNode;917 918 // No nodes? Nothing to do919 if (MaxNodes < 0) {920 DEBUG(report("No nodes!\n"));921 CFGNodes = nullptr;922 SpanningTreeNodes = nullptr;923 NumNodes = 0;924 return;925 }926 ++MaxNodes;927 DEBUG(reportNumber("NumNodes = ", MaxNodes, 10));928 NumNodes = static_cast<uint32_t>(MaxNodes);929 930 // Initial allocations931 CFGNodes = new (Alloc) Node[MaxNodes];932 933 DEBUG(reportNumber("G->CFGNodes = 0x", (uint64_t)CFGNodes, 16));934 SpanningTreeNodes = new (Alloc) Node[MaxNodes];935 DEBUG(reportNumber("G->SpanningTreeNodes = 0x",936 (uint64_t)SpanningTreeNodes, 16));937 938 // Figure out how much to allocate to each vector (in/out edge sets)939 for (int I = 0; I < D.NumEdges; ++I) {940 CFGNodes[D.Edges[I].FromNode].NumOutEdges++;941 CFGNodes[D.Edges[I].ToNode].NumInEdges++;942 if (D.Edges[I].Counter != 0xffffffff)943 continue;944 945 SpanningTreeNodes[D.Edges[I].FromNode].NumOutEdges++;946 SpanningTreeNodes[D.Edges[I].ToNode].NumInEdges++;947 }948 949 // Allocate in/out edge sets950 for (int I = 0; I < MaxNodes; ++I) {951 if (CFGNodes[I].NumInEdges > 0)952 CFGNodes[I].InEdges = new (Alloc) Edge[CFGNodes[I].NumInEdges];953 if (CFGNodes[I].NumOutEdges > 0)954 CFGNodes[I].OutEdges = new (Alloc) Edge[CFGNodes[I].NumOutEdges];955 if (SpanningTreeNodes[I].NumInEdges > 0)956 SpanningTreeNodes[I].InEdges =957 new (Alloc) Edge[SpanningTreeNodes[I].NumInEdges];958 if (SpanningTreeNodes[I].NumOutEdges > 0)959 SpanningTreeNodes[I].OutEdges =960 new (Alloc) Edge[SpanningTreeNodes[I].NumOutEdges];961 CFGNodes[I].NumInEdges = 0;962 CFGNodes[I].NumOutEdges = 0;963 SpanningTreeNodes[I].NumInEdges = 0;964 SpanningTreeNodes[I].NumOutEdges = 0;965 }966 967 // Fill in/out edge sets968 for (int I = 0; I < D.NumEdges; ++I) {969 const uint32_t Src = D.Edges[I].FromNode;970 const uint32_t Dst = D.Edges[I].ToNode;971 Edge *E = &CFGNodes[Src].OutEdges[CFGNodes[Src].NumOutEdges++];972 E->Node = Dst;973 E->ID = I;974 975 E = &CFGNodes[Dst].InEdges[CFGNodes[Dst].NumInEdges++];976 E->Node = Src;977 E->ID = I;978 979 if (D.Edges[I].Counter != 0xffffffff)980 continue;981 982 E = &SpanningTreeNodes[Src]983 .OutEdges[SpanningTreeNodes[Src].NumOutEdges++];984 E->Node = Dst;985 E->ID = I;986 987 E = &SpanningTreeNodes[Dst]988 .InEdges[SpanningTreeNodes[Dst].NumInEdges++];989 E->Node = Src;990 E->ID = I;991 }992 993 computeEdgeFrequencies(Counters, Ctx);994}995 996Graph::~Graph() {997 if (CallFreqs)998 Alloc.deallocate(CallFreqs);999 if (EdgeFreqs)1000 Alloc.deallocate(EdgeFreqs);1001 for (int I = NumNodes - 1; I >= 0; --I) {1002 if (SpanningTreeNodes[I].OutEdges)1003 Alloc.deallocate(SpanningTreeNodes[I].OutEdges);1004 if (SpanningTreeNodes[I].InEdges)1005 Alloc.deallocate(SpanningTreeNodes[I].InEdges);1006 if (CFGNodes[I].OutEdges)1007 Alloc.deallocate(CFGNodes[I].OutEdges);1008 if (CFGNodes[I].InEdges)1009 Alloc.deallocate(CFGNodes[I].InEdges);1010 }1011 if (SpanningTreeNodes)1012 Alloc.deallocate(SpanningTreeNodes);1013 if (CFGNodes)1014 Alloc.deallocate(CFGNodes);1015}1016 1017void Graph::dump() const {1018 reportNumber("Dumping graph with number of nodes: ", NumNodes, 10);1019 report(" Full graph:\n");1020 for (int I = 0; I < NumNodes; ++I) {1021 const Node *N = &CFGNodes[I];1022 reportNumber(" Node #", I, 10);1023 reportNumber(" InEdges total ", N->NumInEdges, 10);1024 for (int J = 0; J < N->NumInEdges; ++J)1025 reportNumber(" ", N->InEdges[J].Node, 10);1026 reportNumber(" OutEdges total ", N->NumOutEdges, 10);1027 for (int J = 0; J < N->NumOutEdges; ++J)1028 reportNumber(" ", N->OutEdges[J].Node, 10);1029 report("\n");1030 }1031 report(" Spanning tree:\n");1032 for (int I = 0; I < NumNodes; ++I) {1033 const Node *N = &SpanningTreeNodes[I];1034 reportNumber(" Node #", I, 10);1035 reportNumber(" InEdges total ", N->NumInEdges, 10);1036 for (int J = 0; J < N->NumInEdges; ++J)1037 reportNumber(" ", N->InEdges[J].Node, 10);1038 reportNumber(" OutEdges total ", N->NumOutEdges, 10);1039 for (int J = 0; J < N->NumOutEdges; ++J)1040 reportNumber(" ", N->OutEdges[J].Node, 10);1041 report("\n");1042 }1043}1044 1045void Graph::dumpEdgeFreqs() const {1046 reportNumber(1047 "Dumping edge frequencies for graph with num edges: ", D.NumEdges, 10);1048 for (int I = 0; I < D.NumEdges; ++I) {1049 reportNumber("* Src: ", D.Edges[I].FromNode, 10);1050 reportNumber(" Dst: ", D.Edges[I].ToNode, 10);1051 reportNumber(" Cnt: ", EdgeFreqs[I], 10);1052 }1053}1054 1055/// Auxiliary map structure for fast lookups of which calls map to each node of1056/// the function CFG1057struct NodeToCallsMap {1058 struct MapEntry {1059 uint32_t NumCalls;1060 uint32_t *Calls;1061 };1062 MapEntry *Entries;1063 BumpPtrAllocator &Alloc;1064 const uint32_t NumNodes;1065 1066 NodeToCallsMap(BumpPtrAllocator &Alloc, const FunctionDescription &D,1067 uint32_t NumNodes)1068 : Alloc(Alloc), NumNodes(NumNodes) {1069 Entries = new (Alloc, 0) MapEntry[NumNodes];1070 for (int I = 0; I < D.NumCalls; ++I) {1071 DEBUG(reportNumber("Registering call in node ", D.Calls[I].FromNode, 10));1072 ++Entries[D.Calls[I].FromNode].NumCalls;1073 }1074 for (int I = 0; I < NumNodes; ++I) {1075 Entries[I].Calls = Entries[I].NumCalls ? new (Alloc)1076 uint32_t[Entries[I].NumCalls]1077 : nullptr;1078 Entries[I].NumCalls = 0;1079 }1080 for (int I = 0; I < D.NumCalls; ++I) {1081 MapEntry &Entry = Entries[D.Calls[I].FromNode];1082 Entry.Calls[Entry.NumCalls++] = I;1083 }1084 }1085 1086 /// Set the frequency of all calls in node \p NodeID to Freq. However, if1087 /// the calls have their own counters and do not depend on the basic block1088 /// counter, this means they have landing pads and throw exceptions. In this1089 /// case, set their frequency with their counters and return the maximum1090 /// value observed in such counters. This will be used as the new frequency1091 /// at basic block entry. This is used to fix the CFG edge frequencies in the1092 /// presence of exceptions.1093 uint64_t visitAllCallsIn(uint32_t NodeID, uint64_t Freq, uint64_t *CallFreqs,1094 const FunctionDescription &D,1095 const uint64_t *Counters,1096 ProfileWriterContext &Ctx) const {1097 const MapEntry &Entry = Entries[NodeID];1098 uint64_t MaxValue = 0ull;1099 for (int I = 0, E = Entry.NumCalls; I != E; ++I) {1100 const uint32_t CallID = Entry.Calls[I];1101 DEBUG(reportNumber(" Setting freq for call ID: ", CallID, 10));1102 const CallDescription &CallDesc = D.Calls[CallID];1103 if (CallDesc.Counter == 0xffffffff) {1104 CallFreqs[CallID] = Freq;1105 DEBUG(reportNumber(" with : ", Freq, 10));1106 } else {1107 const uint64_t CounterVal = Counters[CallDesc.Counter];1108 CallFreqs[CallID] = CounterVal;1109 MaxValue = CounterVal > MaxValue ? CounterVal : MaxValue;1110 DEBUG(reportNumber(" with (private counter) : ", CounterVal, 10));1111 }1112 DEBUG(reportNumber(" Address: 0x", CallDesc.TargetAddress, 16));1113 if (CallFreqs[CallID] > 0)1114 Ctx.CallFlowTable->get(CallDesc.TargetAddress).Calls +=1115 CallFreqs[CallID];1116 }1117 return MaxValue;1118 }1119 1120 ~NodeToCallsMap() {1121 for (int I = NumNodes - 1; I >= 0; --I)1122 if (Entries[I].Calls)1123 Alloc.deallocate(Entries[I].Calls);1124 Alloc.deallocate(Entries);1125 }1126};1127 1128/// Fill an array with the frequency of each edge in the function represented1129/// by G, as well as another array for each call.1130void Graph::computeEdgeFrequencies(const uint64_t *Counters,1131 ProfileWriterContext &Ctx) {1132 if (NumNodes == 0)1133 return;1134 1135 EdgeFreqs = D.NumEdges ? new (Alloc, 0) uint64_t [D.NumEdges] : nullptr;1136 CallFreqs = D.NumCalls ? new (Alloc, 0) uint64_t [D.NumCalls] : nullptr;1137 1138 // Setup a lookup for calls present in each node (BB)1139 NodeToCallsMap *CallMap = new (Alloc) NodeToCallsMap(Alloc, D, NumNodes);1140 1141 // Perform a bottom-up, BFS traversal of the spanning tree in G. Edges in the1142 // spanning tree don't have explicit counters. We must infer their value using1143 // a linear combination of other counters (sum of counters of the outgoing1144 // edges minus sum of counters of the incoming edges).1145 uint32_t *Stack = new (Alloc) uint32_t [NumNodes];1146 uint32_t StackTop = 0;1147 enum Status : uint8_t { S_NEW = 0, S_VISITING, S_VISITED };1148 Status *Visited = new (Alloc, 0) Status[NumNodes];1149 uint64_t *LeafFrequency = new (Alloc, 0) uint64_t[NumNodes];1150 uint64_t *EntryAddress = new (Alloc, 0) uint64_t[NumNodes];1151 1152 // Setup a fast lookup for frequency of leaf nodes, which have special1153 // basic block frequency instrumentation (they are not edge profiled).1154 for (int I = 0; I < D.NumLeafNodes; ++I) {1155 LeafFrequency[D.LeafNodes[I].Node] = Counters[D.LeafNodes[I].Counter];1156 DEBUG({1157 if (Counters[D.LeafNodes[I].Counter] > 0) {1158 reportNumber("Leaf Node# ", D.LeafNodes[I].Node, 10);1159 reportNumber(" Counter: ", Counters[D.LeafNodes[I].Counter], 10);1160 }1161 });1162 }1163 for (int I = 0; I < D.NumEntryNodes; ++I) {1164 EntryAddress[D.EntryNodes[I].Node] = D.EntryNodes[I].Address;1165 DEBUG({1166 reportNumber("Entry Node# ", D.EntryNodes[I].Node, 10);1167 reportNumber(" Address: ", D.EntryNodes[I].Address, 16);1168 });1169 }1170 // Add all root nodes to the stack1171 for (int I = 0; I < NumNodes; ++I)1172 if (SpanningTreeNodes[I].NumInEdges == 0)1173 Stack[StackTop++] = I;1174 1175 // Empty stack?1176 if (StackTop == 0) {1177 DEBUG(report("Empty stack!\n"));1178 Alloc.deallocate(EntryAddress);1179 Alloc.deallocate(LeafFrequency);1180 Alloc.deallocate(Visited);1181 Alloc.deallocate(Stack);1182 CallMap->~NodeToCallsMap();1183 Alloc.deallocate(CallMap);1184 if (CallFreqs)1185 Alloc.deallocate(CallFreqs);1186 if (EdgeFreqs)1187 Alloc.deallocate(EdgeFreqs);1188 EdgeFreqs = nullptr;1189 CallFreqs = nullptr;1190 return;1191 }1192 // Add all known edge counts, will infer the rest1193 for (int I = 0; I < D.NumEdges; ++I) {1194 const uint32_t C = D.Edges[I].Counter;1195 if (C == 0xffffffff) // inferred counter - we will compute its value1196 continue;1197 EdgeFreqs[I] = Counters[C];1198 }1199 1200 while (StackTop > 0) {1201 const uint32_t Cur = Stack[--StackTop];1202 DEBUG({1203 if (Visited[Cur] == S_VISITING)1204 report("(visiting) ");1205 else1206 report("(new) ");1207 reportNumber("Cur: ", Cur, 10);1208 });1209 1210 // This shouldn't happen in a tree1211 assert(Visited[Cur] != S_VISITED, "should not have visited nodes in stack");1212 if (Visited[Cur] == S_NEW) {1213 Visited[Cur] = S_VISITING;1214 Stack[StackTop++] = Cur;1215 assert(StackTop <= NumNodes, "stack grew too large");1216 for (int I = 0, E = SpanningTreeNodes[Cur].NumOutEdges; I < E; ++I) {1217 const uint32_t Succ = SpanningTreeNodes[Cur].OutEdges[I].Node;1218 Stack[StackTop++] = Succ;1219 assert(StackTop <= NumNodes, "stack grew too large");1220 }1221 continue;1222 }1223 Visited[Cur] = S_VISITED;1224 1225 // Establish our node frequency based on outgoing edges, which should all be1226 // resolved by now.1227 int64_t CurNodeFreq = LeafFrequency[Cur];1228 // Not a leaf?1229 if (!CurNodeFreq) {1230 for (int I = 0, E = CFGNodes[Cur].NumOutEdges; I != E; ++I) {1231 const uint32_t SuccEdge = CFGNodes[Cur].OutEdges[I].ID;1232 CurNodeFreq += EdgeFreqs[SuccEdge];1233 }1234 }1235 if (CurNodeFreq < 0)1236 CurNodeFreq = 0;1237 1238 const uint64_t CallFreq = CallMap->visitAllCallsIn(1239 Cur, CurNodeFreq > 0 ? CurNodeFreq : 0, CallFreqs, D, Counters, Ctx);1240 1241 // Exception handling affected our output flow? Fix with calls info1242 DEBUG({1243 if (CallFreq > CurNodeFreq)1244 report("Bumping node frequency with call info\n");1245 });1246 CurNodeFreq = CallFreq > CurNodeFreq ? CallFreq : CurNodeFreq;1247 1248 if (CurNodeFreq > 0) {1249 if (uint64_t Addr = EntryAddress[Cur]) {1250 DEBUG(1251 reportNumber(" Setting flow at entry point address 0x", Addr, 16));1252 DEBUG(reportNumber(" with: ", CurNodeFreq, 10));1253 Ctx.CallFlowTable->get(Addr).Val = CurNodeFreq;1254 }1255 }1256 1257 // No parent? Reached a tree root, limit to call frequency updating.1258 if (SpanningTreeNodes[Cur].NumInEdges == 0)1259 continue;1260 1261 assert(SpanningTreeNodes[Cur].NumInEdges == 1, "must have 1 parent");1262 const uint32_t ParentEdge = SpanningTreeNodes[Cur].InEdges[0].ID;1263 1264 // Calculate parent edge freq.1265 int64_t ParentEdgeFreq = CurNodeFreq;1266 for (int I = 0, E = CFGNodes[Cur].NumInEdges; I != E; ++I) {1267 const uint32_t PredEdge = CFGNodes[Cur].InEdges[I].ID;1268 ParentEdgeFreq -= EdgeFreqs[PredEdge];1269 }1270 1271 // Sometimes the conservative CFG that BOLT builds will lead to incorrect1272 // flow computation. For example, in a BB that transitively calls the exit1273 // syscall, BOLT will add a fall-through successor even though it should not1274 // have any successors. So this block execution will likely be wrong. We1275 // tolerate this imperfection since this case should be quite infrequent.1276 if (ParentEdgeFreq < 0) {1277 DEBUG(dumpEdgeFreqs());1278 DEBUG(report("WARNING: incorrect flow"));1279 ParentEdgeFreq = 0;1280 }1281 DEBUG(reportNumber(" Setting freq for ParentEdge: ", ParentEdge, 10));1282 DEBUG(reportNumber(" with ParentEdgeFreq: ", ParentEdgeFreq, 10));1283 EdgeFreqs[ParentEdge] = ParentEdgeFreq;1284 }1285 1286 Alloc.deallocate(EntryAddress);1287 Alloc.deallocate(LeafFrequency);1288 Alloc.deallocate(Visited);1289 Alloc.deallocate(Stack);1290 CallMap->~NodeToCallsMap();1291 Alloc.deallocate(CallMap);1292 DEBUG(dumpEdgeFreqs());1293}1294 1295/// Write to \p FD all of the edge profiles for function \p FuncDesc. Uses1296/// \p Alloc to allocate helper dynamic structures used to compute profile for1297/// edges that we do not explicitly instrument.1298const uint8_t *writeFunctionProfile(int FD, ProfileWriterContext &Ctx,1299 const uint8_t *FuncDesc,1300 BumpPtrAllocator &Alloc) {1301 const FunctionDescription F(FuncDesc);1302 const uint8_t *next = FuncDesc + F.getSize();1303 1304#if !defined(__APPLE__)1305 uint64_t *bolt_instr_locations = __bolt_instr_locations;1306#else1307 uint64_t *bolt_instr_locations = _bolt_instr_locations_getter();1308#endif1309 1310 // Skip funcs we know are cold1311#ifndef ENABLE_DEBUG1312 uint64_t CountersFreq = 0;1313 for (int I = 0; I < F.NumLeafNodes; ++I)1314 CountersFreq += bolt_instr_locations[F.LeafNodes[I].Counter];1315 1316 if (CountersFreq == 0) {1317 for (int I = 0; I < F.NumEdges; ++I) {1318 const uint32_t C = F.Edges[I].Counter;1319 if (C == 0xffffffff)1320 continue;1321 CountersFreq += bolt_instr_locations[C];1322 }1323 if (CountersFreq == 0) {1324 for (int I = 0; I < F.NumCalls; ++I) {1325 const uint32_t C = F.Calls[I].Counter;1326 if (C == 0xffffffff)1327 continue;1328 CountersFreq += bolt_instr_locations[C];1329 }1330 if (CountersFreq == 0)1331 return next;1332 }1333 }1334#endif1335 1336 Graph *G = new (Alloc) Graph(Alloc, F, bolt_instr_locations, Ctx);1337 DEBUG(G->dump());1338 1339 if (!G->EdgeFreqs && !G->CallFreqs) {1340 G->~Graph();1341 Alloc.deallocate(G);1342 return next;1343 }1344 1345 for (int I = 0; I < F.NumEdges; ++I) {1346 const uint64_t Freq = G->EdgeFreqs[I];1347 if (Freq == 0)1348 continue;1349 const EdgeDescription *Desc = &F.Edges[I];1350 char LineBuf[BufSize];1351 char *Ptr = LineBuf;1352 Ptr = serializeLoc(Ctx, Ptr, Desc->From, BufSize);1353 Ptr = serializeLoc(Ctx, Ptr, Desc->To, BufSize - (Ptr - LineBuf));1354 Ptr = strCopy(Ptr, "0 ", BufSize - (Ptr - LineBuf) - 22);1355 Ptr = intToStr(Ptr, Freq, 10);1356 *Ptr++ = '\n';1357 __write(FD, LineBuf, Ptr - LineBuf);1358 }1359 1360 for (int I = 0; I < F.NumCalls; ++I) {1361 const uint64_t Freq = G->CallFreqs[I];1362 if (Freq == 0)1363 continue;1364 char LineBuf[BufSize];1365 char *Ptr = LineBuf;1366 const CallDescription *Desc = &F.Calls[I];1367 Ptr = serializeLoc(Ctx, Ptr, Desc->From, BufSize);1368 Ptr = serializeLoc(Ctx, Ptr, Desc->To, BufSize - (Ptr - LineBuf));1369 Ptr = strCopy(Ptr, "0 ", BufSize - (Ptr - LineBuf) - 25);1370 Ptr = intToStr(Ptr, Freq, 10);1371 *Ptr++ = '\n';1372 __write(FD, LineBuf, Ptr - LineBuf);1373 }1374 1375 G->~Graph();1376 Alloc.deallocate(G);1377 return next;1378}1379 1380#if !defined(__APPLE__)1381const IndCallTargetDescription *1382ProfileWriterContext::lookupIndCallTarget(uint64_t Target) const {1383 uint32_t B = 0;1384 uint32_t E = __bolt_instr_num_ind_targets;1385 if (E == 0)1386 return nullptr;1387 do {1388 uint32_t I = (E - B) / 2 + B;1389 if (IndCallTargets[I].Address == Target)1390 return &IndCallTargets[I];1391 if (IndCallTargets[I].Address < Target)1392 B = I + 1;1393 else1394 E = I;1395 } while (B < E);1396 return nullptr;1397}1398 1399/// Write a single indirect call <src, target> pair to the fdata file1400void visitIndCallCounter(IndirectCallHashTable::MapEntry &Entry,1401 int FD, int CallsiteID,1402 ProfileWriterContext *Ctx) {1403 if (Entry.Val == 0)1404 return;1405 DEBUG(reportNumber("Target func 0x", Entry.Key, 16));1406 DEBUG(reportNumber("Target freq: ", Entry.Val, 10));1407 const IndCallDescription *CallsiteDesc =1408 &Ctx->IndCallDescriptions[CallsiteID];1409 const IndCallTargetDescription *TargetDesc =1410 Ctx->lookupIndCallTarget(Entry.Key - TextBaseAddress);1411 if (!TargetDesc) {1412 DEBUG(report("Failed to lookup indirect call target\n"));1413 char LineBuf[BufSize];1414 char *Ptr = LineBuf;1415 Ptr = serializeLoc(*Ctx, Ptr, *CallsiteDesc, BufSize);1416 Ptr = strCopy(Ptr, "0 [unknown] 0 0 ", BufSize - (Ptr - LineBuf) - 40);1417 Ptr = intToStr(Ptr, Entry.Val, 10);1418 *Ptr++ = '\n';1419 __write(FD, LineBuf, Ptr - LineBuf);1420 return;1421 }1422 Ctx->CallFlowTable->get(TargetDesc->Address).Calls += Entry.Val;1423 char LineBuf[BufSize];1424 char *Ptr = LineBuf;1425 Ptr = serializeLoc(*Ctx, Ptr, *CallsiteDesc, BufSize);1426 Ptr = serializeLoc(*Ctx, Ptr, TargetDesc->Loc, BufSize - (Ptr - LineBuf));1427 Ptr = strCopy(Ptr, "0 ", BufSize - (Ptr - LineBuf) - 25);1428 Ptr = intToStr(Ptr, Entry.Val, 10);1429 *Ptr++ = '\n';1430 __write(FD, LineBuf, Ptr - LineBuf);1431}1432 1433/// Write to \p FD all of the indirect call profiles.1434void writeIndirectCallProfile(int FD, ProfileWriterContext &Ctx) {1435 for (int I = 0; I < __bolt_instr_num_ind_calls; ++I) {1436 DEBUG(reportNumber("IndCallsite #", I, 10));1437 GlobalIndCallCounters[I].forEachElement(visitIndCallCounter, FD, I, &Ctx);1438 }1439}1440 1441/// Check a single call flow for a callee versus all known callers. If there are1442/// less callers than what the callee expects, write the difference with source1443/// [unknown] in the profile.1444void visitCallFlowEntry(CallFlowHashTable::MapEntry &Entry, int FD,1445 ProfileWriterContext *Ctx) {1446 DEBUG(reportNumber("Call flow entry address: 0x", Entry.Key, 16));1447 DEBUG(reportNumber("Calls: ", Entry.Calls, 10));1448 DEBUG(reportNumber("Reported entry frequency: ", Entry.Val, 10));1449 DEBUG({1450 if (Entry.Calls > Entry.Val)1451 report(" More calls than expected!\n");1452 });1453 if (Entry.Val <= Entry.Calls)1454 return;1455 DEBUG(reportNumber(1456 " Balancing calls with traffic: ", Entry.Val - Entry.Calls, 10));1457 const IndCallTargetDescription *TargetDesc =1458 Ctx->lookupIndCallTarget(Entry.Key);1459 if (!TargetDesc) {1460 // There is probably something wrong with this callee and this should be1461 // investigated, but I don't want to assert and lose all data collected.1462 DEBUG(report("WARNING: failed to look up call target!\n"));1463 return;1464 }1465 char LineBuf[BufSize];1466 char *Ptr = LineBuf;1467 Ptr = strCopy(Ptr, "0 [unknown] 0 ", BufSize);1468 Ptr = serializeLoc(*Ctx, Ptr, TargetDesc->Loc, BufSize - (Ptr - LineBuf));1469 Ptr = strCopy(Ptr, "0 ", BufSize - (Ptr - LineBuf) - 25);1470 Ptr = intToStr(Ptr, Entry.Val - Entry.Calls, 10);1471 *Ptr++ = '\n';1472 __write(FD, LineBuf, Ptr - LineBuf);1473}1474 1475/// Open fdata file for writing and return a valid file descriptor, aborting1476/// program upon failure.1477int openProfile() {1478 // Build the profile name string by appending our PID1479 char Buf[BufSize];1480 uint64_t PID = __getpid();1481 char *Ptr = strCopy(Buf, __bolt_instr_filename, BufSize);1482 if (__bolt_instr_use_pid) {1483 Ptr = strCopy(Ptr, ".", BufSize - (Ptr - Buf + 1));1484 Ptr = intToStr(Ptr, PID, 10);1485 Ptr = strCopy(Ptr, ".fdata", BufSize - (Ptr - Buf + 1));1486 }1487 *Ptr++ = '\0';1488 uint64_t FD = __open(Buf, O_WRONLY | O_TRUNC | O_CREAT,1489 /*mode=*/0666);1490 if (static_cast<int64_t>(FD) < 0) {1491 report("Error while trying to open profile file for writing: ");1492 report(Buf);1493 reportNumber("\nFailed with error number: 0x",1494 0 - static_cast<int64_t>(FD), 16);1495 __exit(1);1496 }1497 return FD;1498}1499 1500#endif1501 1502} // anonymous namespace1503 1504#if !defined(__APPLE__)1505 1506/// Reset all counters in case you want to start profiling a new phase of your1507/// program independently of prior phases.1508/// The address of this function is printed by BOLT and this can be called by1509/// any attached debugger during runtime. There is a useful oneliner for gdb:1510///1511/// gdb -p $(pgrep -xo PROCESSNAME) -ex 'p ((void(*)())0xdeadbeef)()' \1512/// -ex 'set confirm off' -ex quit1513///1514/// Where 0xdeadbeef is this function address and PROCESSNAME your binary file1515/// name.1516extern "C" void __bolt_instr_clear_counters() {1517 memset(reinterpret_cast<char *>(__bolt_instr_locations), 0,1518 __bolt_num_counters * 8);1519 for (int I = 0; I < __bolt_instr_num_ind_calls; ++I)1520 GlobalIndCallCounters[I].resetCounters();1521}1522 1523/// This is the entry point for profile writing.1524/// There are four ways of getting here:1525///1526/// * Program execution ended, finalization methods are running and BOLT1527/// hooked into FINI from your binary dynamic section;1528/// * You used the sleep timer option and during initialization we forked1529/// a separate process that will call this function periodically;1530/// * BOLT prints this function address so you can attach a debugger and1531/// call this function directly to get your profile written to disk1532/// on demand.1533/// * Application can, at interesting runtime point, iterate through all1534/// the loaded native libraries and for each call dlopen() and dlsym()1535/// to get a pointer to this function and call through the acquired1536/// function pointer to dump profile data.1537///1538extern "C" void __attribute((force_align_arg_pointer))1539__bolt_instr_data_dump(int FD, const char *LibPath = nullptr,1540 const uint8_t *LibContents = nullptr,1541 uint64_t LibSize = 0) {1542 if (LibPath)1543 strCopy(TargetPath, LibPath, NameMax);1544 1545 // Already dumping1546 if (!GlobalWriteProfileMutex->acquire())1547 return;1548 1549 int ret = __lseek(FD, 0, SEEK_SET);1550 assert(ret == 0, "Failed to lseek!");1551 ret = __ftruncate(FD, 0);1552 assert(ret == 0, "Failed to ftruncate!");1553 BumpPtrAllocator HashAlloc;1554 HashAlloc.setMaxSize(0x6400000);1555 ProfileWriterContext Ctx = readDescriptions(LibContents, LibSize);1556 Ctx.CallFlowTable = new (HashAlloc, 0) CallFlowHashTable(HashAlloc);1557 1558 DEBUG(printStats(Ctx));1559 1560 BumpPtrAllocator Alloc;1561 Alloc.setMaxSize(0x6400000);1562 const uint8_t *FuncDesc = Ctx.FuncDescriptions;1563 for (int I = 0, E = __bolt_instr_num_funcs; I < E; ++I) {1564 FuncDesc = writeFunctionProfile(FD, Ctx, FuncDesc, Alloc);1565 Alloc.clear();1566 DEBUG(reportNumber("FuncDesc now: ", (uint64_t)FuncDesc, 16));1567 }1568 assert(FuncDesc == (void *)Ctx.Strings,1569 "FuncDesc ptr must be equal to stringtable");1570 1571 writeIndirectCallProfile(FD, Ctx);1572 Ctx.CallFlowTable->forEachElement(visitCallFlowEntry, FD, &Ctx);1573 1574 __fsync(FD);1575 if (Ctx.FileDesc != -1) {1576 __munmap((void *)Ctx.MMapPtr, Ctx.MMapSize);1577 __close(Ctx.FileDesc);1578 }1579 HashAlloc.destroy();1580 GlobalWriteProfileMutex->release();1581 DEBUG(report("Finished writing profile.\n"));1582}1583 1584/// Event loop for our child process spawned during setup to dump profile data1585/// at user-specified intervals1586void watchProcess() {1587 timespec ts, rem;1588 uint64_t Elapsed = 0ull;1589 int FD = openProfile();1590 uint64_t ppid;1591 if (__bolt_instr_wait_forks) {1592 // Store parent pgid1593 ppid = -__getpgid(0);1594 // And leave parent process group1595 __setpgid(0, 0);1596 } else {1597 // Store parent pid1598 ppid = __getppid();1599 if (ppid == 1) {1600 // Parent already dead1601 __bolt_instr_data_dump(FD);1602 goto out;1603 }1604 }1605 1606 ts.tv_sec = 1;1607 ts.tv_nsec = 0;1608 while (1) {1609 __nanosleep(&ts, &rem);1610 // This means our parent process or all its forks are dead,1611 // so no need for us to keep dumping.1612 if (__kill(ppid, 0) < 0) {1613 if (__bolt_instr_no_counters_clear)1614 __bolt_instr_data_dump(FD);1615 break;1616 }1617 1618 if (++Elapsed < __bolt_instr_sleep_time)1619 continue;1620 1621 Elapsed = 0;1622 __bolt_instr_data_dump(FD);1623 if (__bolt_instr_no_counters_clear == false)1624 __bolt_instr_clear_counters();1625 }1626 1627out:;1628 DEBUG(report("My parent process is dead, bye!\n"));1629 __close(FD);1630 __exit(0);1631}1632 1633extern "C" void __bolt_instr_indirect_call();1634extern "C" void __bolt_instr_indirect_tailcall();1635 1636/// Initialization code1637extern "C" void __attribute((force_align_arg_pointer)) __bolt_instr_setup() {1638 __bolt_ind_call_counter_func_pointer = __bolt_instr_indirect_call;1639 __bolt_ind_tailcall_counter_func_pointer = __bolt_instr_indirect_tailcall;1640 TextBaseAddress = getTextBaseAddress();1641 1642 const uint64_t CountersStart =1643 reinterpret_cast<uint64_t>(&__bolt_instr_locations[0]);1644 const uint64_t CountersEnd = alignTo(1645 reinterpret_cast<uint64_t>(&__bolt_instr_locations[__bolt_num_counters]),1646 0x1000);1647 DEBUG(reportNumber("replace mmap start: ", CountersStart, 16));1648 DEBUG(reportNumber("replace mmap stop: ", CountersEnd, 16));1649 assert(CountersEnd > CountersStart, "no counters");1650 1651 const bool Shared = !__bolt_instr_use_pid;1652 const uint64_t MapPrivateOrShared = Shared ? MAP_SHARED : MAP_PRIVATE;1653 1654 void *Ret =1655 __mmap(CountersStart, CountersEnd - CountersStart, PROT_READ | PROT_WRITE,1656 MAP_ANONYMOUS | MapPrivateOrShared | MAP_FIXED, -1, 0);1657 assert(Ret != MAP_FAILED, "__bolt_instr_setup: Failed to mmap counters!");1658 1659 GlobalMetadataStorage = __mmap(0, 4096, PROT_READ | PROT_WRITE,1660 MapPrivateOrShared | MAP_ANONYMOUS, -1, 0);1661 assert(GlobalMetadataStorage != MAP_FAILED,1662 "__bolt_instr_setup: failed to mmap page for metadata!");1663 1664 GlobalAlloc = new (GlobalMetadataStorage) BumpPtrAllocator;1665 // Conservatively reserve 100MiB1666 GlobalAlloc->setMaxSize(0x6400000);1667 GlobalAlloc->setShared(Shared);1668 GlobalWriteProfileMutex = new (*GlobalAlloc, 0) Mutex();1669 if (__bolt_instr_num_ind_calls > 0)1670 GlobalIndCallCounters =1671 new (*GlobalAlloc, 0) IndirectCallHashTable[__bolt_instr_num_ind_calls];1672 1673 if (__bolt_instr_sleep_time != 0) {1674 // Separate instrumented process to the own process group1675 if (__bolt_instr_wait_forks)1676 __setpgid(0, 0);1677 1678 if (long PID = __fork())1679 return;1680 watchProcess();1681 }1682}1683 1684extern "C" __attribute((force_align_arg_pointer)) void1685instrumentIndirectCall(uint64_t Target, uint64_t IndCallID) {1686 GlobalIndCallCounters[IndCallID].incrementVal(Target, *GlobalAlloc);1687}1688 1689/// We receive as in-stack arguments the identifier of the indirect call site1690/// as well as the target address for the call1691extern "C" __attribute((naked)) void __bolt_instr_indirect_call()1692{1693#if defined(__aarch64__)1694 // the target address is placed on stack1695 // the identifier of the indirect call site is placed in X1 register1696 1697 // clang-format off1698 __asm__ __volatile__(SAVE_ALL1699 "ldr x0, [sp, #272]\n"1700 "bl instrumentIndirectCall\n"1701 RESTORE_ALL1702 "ret\n"1703 :::);1704 // clang-format on1705#elif defined(__riscv)1706 // clang-format off1707 __asm__ __volatile__(1708 SAVE_ALL1709 "addi sp, sp, 288\n"1710 "ld x10, 0(sp)\n"1711 "ld x11, 8(sp)\n"1712 "addi sp, sp, -288\n"1713 "jal x1, instrumentIndirectCall\n"1714 RESTORE_ALL1715 "ret\n"1716 :::);1717 // clang-format on1718#else1719 // clang-format off1720 __asm__ __volatile__(SAVE_ALL1721 "mov 0xa0(%%rsp), %%rdi\n"1722 "mov 0x98(%%rsp), %%rsi\n"1723 "call instrumentIndirectCall\n"1724 RESTORE_ALL1725 "ret\n"1726 :::);1727 // clang-format on1728#endif1729}1730 1731extern "C" __attribute((naked)) void __bolt_instr_indirect_tailcall()1732{1733#if defined(__aarch64__)1734 // the target address is placed on stack1735 // the identifier of the indirect call site is placed in X1 register1736 1737 // clang-format off1738 __asm__ __volatile__(SAVE_ALL1739 "ldr x0, [sp, #272]\n"1740 "bl instrumentIndirectCall\n"1741 RESTORE_ALL1742 "ret\n"1743 :::);1744 // clang-format on1745#elif defined(__riscv)1746 // clang-format off1747 __asm__ __volatile__(SAVE_ALL1748 "addi sp, sp, 288\n"1749 "ld x10, 0(sp)\n"1750 "ld x11, 8(sp)\n"1751 "addi sp, sp, -288\n"1752 "jal x1, instrumentIndirectCall\n"1753 RESTORE_ALL1754 "ret\n"1755 :::);1756 // clang-format on1757#else1758 // clang-format off1759 __asm__ __volatile__(SAVE_ALL1760 "mov 0x98(%%rsp), %%rdi\n"1761 "mov 0x90(%%rsp), %%rsi\n"1762 "call instrumentIndirectCall\n"1763 RESTORE_ALL1764 "ret\n"1765 :::);1766 // clang-format on1767#endif1768}1769 1770/// This is hooking ELF's entry, it needs to save all machine state.1771extern "C" __attribute((naked)) void __bolt_instr_start()1772{1773#if defined(__aarch64__)1774 // clang-format off1775 __asm__ __volatile__(SAVE_ALL1776 "bl __bolt_instr_setup\n"1777 RESTORE_ALL1778 "adrp x16, __bolt_start_trampoline\n"1779 "add x16, x16, #:lo12:__bolt_start_trampoline\n"1780 "br x16\n"1781 :::);1782 // clang-format on1783#elif defined(__riscv)1784 // clang-format off1785 __asm__ __volatile__(1786 SAVE_ALL1787 "jal x1, __bolt_instr_setup\n"1788 RESTORE_ALL1789 "setup_symbol:\n"1790 "auipc x5, %%pcrel_hi(__bolt_start_trampoline)\n"1791 "addi x5, x5, %%pcrel_lo(setup_symbol)\n"1792 "jr x5\n"1793 :::);1794 // clang-format on1795#else1796 // clang-format off1797 __asm__ __volatile__(SAVE_ALL1798 "call __bolt_instr_setup\n"1799 RESTORE_ALL1800 "jmp __bolt_start_trampoline\n"1801 :::);1802 // clang-format on1803#endif1804}1805 1806/// This is hooking into ELF's DT_FINI1807extern "C" void __bolt_instr_fini() {1808#if defined(__aarch64__)1809 // clang-format off1810 __asm__ __volatile__(SAVE_ALL1811 "adrp x16, __bolt_fini_trampoline\n"1812 "add x16, x16, #:lo12:__bolt_fini_trampoline\n"1813 "blr x16\n"1814 RESTORE_ALL1815 :::);1816 // clang-format on1817#elif defined(__riscv)1818 // clang-format off1819 __asm__ __volatile__(1820 SAVE_ALL1821 "fini_symbol:\n"1822 "auipc x5, %%pcrel_hi(__bolt_fini_trampoline)\n"1823 "addi x5, x5, %%pcrel_lo(fini_symbol)\n"1824 "jalr x1, 0(x5)\n"1825 RESTORE_ALL1826 :::);1827 // clang-format on1828#else1829 __asm__ __volatile__("call __bolt_fini_trampoline\n" :::);1830#endif1831 if (__bolt_instr_sleep_time == 0) {1832 int FD = openProfile();1833 __bolt_instr_data_dump(FD);1834 __close(FD);1835 }1836 DEBUG(report("Finished.\n"));1837}1838 1839#endif1840 1841#if defined(__APPLE__)1842 1843extern "C" void __bolt_instr_data_dump() {1844 ProfileWriterContext Ctx = readDescriptions();1845 1846 int FD = 2;1847 BumpPtrAllocator Alloc;1848 const uint8_t *FuncDesc = Ctx.FuncDescriptions;1849 uint32_t bolt_instr_num_funcs = _bolt_instr_num_funcs_getter();1850 1851 for (int I = 0, E = bolt_instr_num_funcs; I < E; ++I) {1852 FuncDesc = writeFunctionProfile(FD, Ctx, FuncDesc, Alloc);1853 Alloc.clear();1854 DEBUG(reportNumber("FuncDesc now: ", (uint64_t)FuncDesc, 16));1855 }1856 assert(FuncDesc == (void *)Ctx.Strings,1857 "FuncDesc ptr must be equal to stringtable");1858}1859 1860// On OSX/iOS the final symbol name of an extern "C" function/variable contains1861// one extra leading underscore: _bolt_instr_setup -> __bolt_instr_setup.1862extern "C"1863__attribute__((section("__TEXT,__setup")))1864__attribute__((force_align_arg_pointer))1865void _bolt_instr_setup() {1866 __asm__ __volatile__(SAVE_ALL :::);1867 1868 report("Hello!\n");1869 1870 __asm__ __volatile__(RESTORE_ALL :::);1871}1872 1873extern "C"1874__attribute__((section("__TEXT,__fini")))1875__attribute__((force_align_arg_pointer))1876void _bolt_instr_fini() {1877 report("Bye!\n");1878 __bolt_instr_data_dump();1879}1880 1881#endif1882