298 lines · cpp
1#include <stdint.h>2#include <stdlib.h>3#include <string.h>4 5#include "memprof_rawprofile.h"6#include "profile/MemProfData.inc"7#include "sanitizer_common/sanitizer_allocator_internal.h"8#include "sanitizer_common/sanitizer_array_ref.h"9#include "sanitizer_common/sanitizer_common.h"10#include "sanitizer_common/sanitizer_stackdepot.h"11#include "sanitizer_common/sanitizer_stacktrace.h"12#include "sanitizer_common/sanitizer_vector.h"13 14namespace __memprof {15using ::__sanitizer::Vector;16using ::llvm::memprof::MemInfoBlock;17using SegmentEntry = ::llvm::memprof::SegmentEntry;18using Header = ::llvm::memprof::Header;19using ::llvm::memprof::encodeHistogramCount;20 21namespace {22template <class T> char *WriteBytes(const T &Pod, char *Buffer) {23 static_assert(is_trivially_copyable<T>::value, "T must be POD");24 const uint8_t *Src = reinterpret_cast<const uint8_t *>(&Pod);25 26 for (size_t I = 0; I < sizeof(T); ++I)27#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__28 // Reverse byte order since reader is little-endian.29 Buffer[I] = Src[sizeof(T) - 1 - I];30#else31 Buffer[I] = Src[I];32#endif33 return Buffer + sizeof(T);34}35 36void RecordStackId(const uptr Key, UNUSED LockedMemInfoBlock *const &MIB,37 void *Arg) {38 // No need to touch the MIB value here since we are only recording the key.39 auto *StackIds = reinterpret_cast<Vector<u64> *>(Arg);40 StackIds->PushBack(Key);41}42 43u64 SegmentSizeBytes(ArrayRef<LoadedModule> Modules) {44 u64 NumSegmentsToRecord = 0;45 for (const auto &Module : Modules) {46 for (const auto &Segment : Module.ranges()) {47 if (Segment.executable)48 NumSegmentsToRecord++;49 }50 }51 52 return sizeof(u64) // A header which stores the number of records.53 + sizeof(SegmentEntry) * NumSegmentsToRecord;54}55 56// The segment section uses the following format:57// ---------- Segment Info58// Num Entries59// ---------- Segment Entry60// Start61// End62// Offset63// UuidSize64// Uuid 32B65// ----------66// ...67void SerializeSegmentsToBuffer(ArrayRef<LoadedModule> Modules,68 const u64 ExpectedNumBytes, char *&Buffer) {69 char *Ptr = Buffer;70 // Reserve space for the final count.71 Ptr += sizeof(u64);72 73 u64 NumSegmentsRecorded = 0;74 75 for (const auto &Module : Modules) {76 for (const auto &Segment : Module.ranges()) {77 if (Segment.executable) {78 SegmentEntry Entry(Segment.beg, Segment.end, Module.base_address());79 CHECK(Module.uuid_size() <= MEMPROF_BUILDID_MAX_SIZE);80 Entry.BuildIdSize = Module.uuid_size();81 memcpy(Entry.BuildId, Module.uuid(), Module.uuid_size());82 memcpy(Ptr, &Entry, sizeof(SegmentEntry));83 Ptr += sizeof(SegmentEntry);84 NumSegmentsRecorded++;85 }86 }87 }88 // Store the number of segments we recorded in the space we reserved.89 *((u64 *)Buffer) = NumSegmentsRecorded;90 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&91 "Expected num bytes != actual bytes written");92}93 94u64 StackSizeBytes(const Vector<u64> &StackIds) {95 u64 NumBytesToWrite = sizeof(u64);96 97 const u64 NumIds = StackIds.Size();98 for (unsigned k = 0; k < NumIds; ++k) {99 const u64 Id = StackIds[k];100 // One entry for the id and then one more for the number of stack pcs.101 NumBytesToWrite += 2 * sizeof(u64);102 const StackTrace St = StackDepotGet(Id);103 104 CHECK(St.trace != nullptr && St.size > 0 && "Empty stack trace");105 for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {106 NumBytesToWrite += sizeof(u64);107 }108 }109 return NumBytesToWrite;110}111 112// The stack info section uses the following format:113//114// ---------- Stack Info115// Num Entries116// ---------- Stack Entry117// Num Stacks118// PC1119// PC2120// ...121// ----------122void SerializeStackToBuffer(const Vector<u64> &StackIds,123 const u64 ExpectedNumBytes, char *&Buffer) {124 const u64 NumIds = StackIds.Size();125 char *Ptr = Buffer;126 Ptr = WriteBytes(static_cast<u64>(NumIds), Ptr);127 128 for (unsigned k = 0; k < NumIds; ++k) {129 const u64 Id = StackIds[k];130 Ptr = WriteBytes(Id, Ptr);131 Ptr += sizeof(u64); // Bump it by u64, we will fill this in later.132 u64 Count = 0;133 const StackTrace St = StackDepotGet(Id);134 for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) {135 // PCs in stack traces are actually the return addresses, that is,136 // addresses of the next instructions after the call.137 uptr pc = StackTrace::GetPreviousInstructionPc(St.trace[i]);138 Ptr = WriteBytes(static_cast<u64>(pc), Ptr);139 ++Count;140 }141 // Store the count in the space we reserved earlier.142 *(u64 *)(Ptr - (Count + 1) * sizeof(u64)) = Count;143 }144 145 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&146 "Expected num bytes != actual bytes written");147}148 149// The MIB section has the following format:150// ---------- MIB Info151// Num Entries152// ---------- MIB Entry 0153// Alloc Count154// ...155// ---- AccessHistogram Entry 0156// ...157// ---- AccessHistogram Entry AccessHistogramSize - 1158// ---------- MIB Entry 1159// Alloc Count160// ...161// ---- AccessHistogram Entry 0162// ...163// ---- AccessHistogram Entry AccessHistogramSize - 1164// ----------165void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds,166 const u64 ExpectedNumBytes, char *&Buffer) {167 char *Ptr = Buffer;168 const u64 NumEntries = StackIds.Size();169 Ptr = WriteBytes(NumEntries, Ptr);170 for (u64 i = 0; i < NumEntries; i++) {171 const u64 Key = StackIds[i];172 MIBMapTy::Handle h(&MIBMap, Key, /*remove=*/true, /*create=*/false);173 CHECK(h.exists());174 Ptr = WriteBytes(Key, Ptr);175 // FIXME: We unnecessarily serialize the AccessHistogram pointer. Adding a176 // serialization schema will fix this issue. See also FIXME in177 // deserialization.178 auto &MIB = (*h)->mib;179 Ptr = WriteBytes(MIB, Ptr);180 for (u64 j = 0; j < MIB.AccessHistogramSize; ++j) {181 u16 HistogramEntry =182 encodeHistogramCount(((u64 *)(MIB.AccessHistogram))[j]);183 Ptr = WriteBytes(HistogramEntry, Ptr);184 }185 if (MIB.AccessHistogramSize > 0) {186 InternalFree((void *)MIB.AccessHistogram);187 }188 }189 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) &&190 "Expected num bytes != actual bytes written");191}192} // namespace193 194// Format195// ---------- Header196// Magic197// Version198// Total Size199// Segment Offset200// MIB Info Offset201// Stack Offset202// ---------- Segment Info203// Num Entries204// ---------- Segment Entry205// Start206// End207// Offset208// BuildID 32B209// ----------210// ...211// ----------212// Optional Padding Bytes213// ---------- MIB Info214// Num Entries215// ---------- MIB Entry216// Alloc Count217// ...218// ---- AccessHistogram Entry 0219// ...220// ---- AccessHistogram Entry AccessHistogramSize - 1221// ---------- MIB Entry 1222// Alloc Count223// ...224// ---- AccessHistogram Entry 0225// ...226// ---- AccessHistogram Entry AccessHistogramSize - 1227// Optional Padding Bytes228// ---------- Stack Info229// Num Entries230// ---------- Stack Entry231// Num Stacks232// PC1233// PC2234// ...235// ----------236// Optional Padding Bytes237// ...238u64 SerializeToRawProfile(MIBMapTy &MIBMap, ArrayRef<LoadedModule> Modules,239 char *&Buffer) {240 // Each section size is rounded up to 8b since the first entry in each section241 // is a u64 which holds the number of entries in the section by convention.242 const u64 NumSegmentBytes = RoundUpTo(SegmentSizeBytes(Modules), 8);243 244 Vector<u64> StackIds;245 MIBMap.ForEach(RecordStackId, reinterpret_cast<void *>(&StackIds));246 // The first 8b are for the total number of MIB records. Each MIB record is247 // preceded by a 8b stack id which is associated with stack frames in the next248 // section.249 const u64 NumMIBInfoBytes = RoundUpTo(250 sizeof(u64) + StackIds.Size() * (sizeof(u64) + sizeof(MemInfoBlock)), 8);251 252 // Get Number of AccessHistogram entries in total253 u64 TotalAccessHistogramEntries = 0;254 MIBMap.ForEach(255 [](const uptr Key, UNUSED LockedMemInfoBlock *const &MIB, void *Arg) {256 u64 *TotalAccessHistogramEntries = (u64 *)Arg;257 *TotalAccessHistogramEntries += MIB->mib.AccessHistogramSize;258 },259 reinterpret_cast<void *>(&TotalAccessHistogramEntries));260 const u64 NumHistogramBytes =261 RoundUpTo(TotalAccessHistogramEntries * sizeof(uint16_t), 8);262 263 const u64 NumStackBytes = RoundUpTo(StackSizeBytes(StackIds), 8);264 265 // Ensure that the profile is 8b aligned. We allow for some optional padding266 // at the end so that any subsequent profile serialized to the same file does267 // not incur unaligned accesses.268 const u64 TotalSizeBytes =269 RoundUpTo(sizeof(Header) + NumSegmentBytes + NumStackBytes +270 NumMIBInfoBytes + NumHistogramBytes,271 8);272 273 // Allocate the memory for the entire buffer incl. info blocks.274 Buffer = (char *)InternalAlloc(TotalSizeBytes);275 char *Ptr = Buffer;276 277 Header header{MEMPROF_RAW_MAGIC_64,278 MEMPROF_RAW_VERSION,279 static_cast<u64>(TotalSizeBytes),280 sizeof(Header),281 sizeof(Header) + NumSegmentBytes,282 sizeof(Header) + NumSegmentBytes + NumMIBInfoBytes +283 NumHistogramBytes};284 Ptr = WriteBytes(header, Ptr);285 286 SerializeSegmentsToBuffer(Modules, NumSegmentBytes, Ptr);287 Ptr += NumSegmentBytes;288 289 SerializeMIBInfoToBuffer(MIBMap, StackIds,290 NumMIBInfoBytes + NumHistogramBytes, Ptr);291 Ptr += NumMIBInfoBytes + NumHistogramBytes;292 293 SerializeStackToBuffer(StackIds, NumStackBytes, Ptr);294 295 return TotalSizeBytes;296}297} // namespace __memprof298