412 lines · cpp
1//===-- xray_profile_collector.cpp -----------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a part of XRay, a dynamic runtime instrumentation system.10//11// This implements the interface for the profileCollectorService.12//13//===----------------------------------------------------------------------===//14#include "xray_profile_collector.h"15#include "sanitizer_common/sanitizer_common.h"16#include "xray_allocator.h"17#include "xray_defs.h"18#include "xray_profiling_flags.h"19#include "xray_segmented_array.h"20#include <memory>21#include <pthread.h>22#include <utility>23 24namespace __xray {25namespace profileCollectorService {26 27namespace {28 29SpinMutex GlobalMutex;30struct ThreadTrie {31 ThreadID TId;32 alignas(FunctionCallTrie) std::byte TrieStorage[sizeof(FunctionCallTrie)];33};34 35struct ProfileBuffer {36 void *Data;37 size_t Size;38};39 40// Current version of the profile format.41constexpr u64 XRayProfilingVersion = 0x20180424;42 43// Identifier for XRay profiling files 'xrayprof' in hex.44constexpr u64 XRayMagicBytes = 0x7872617970726f66;45 46struct XRayProfilingFileHeader {47 const u64 MagicBytes = XRayMagicBytes;48 const u64 Version = XRayProfilingVersion;49 u64 Timestamp = 0; // System time in nanoseconds.50 u64 PID = 0; // Process ID.51};52 53struct BlockHeader {54 u32 BlockSize;55 u32 BlockNum;56 u64 ThreadId;57};58 59struct ThreadData {60 BufferQueue *BQ;61 FunctionCallTrie::Allocators::Buffers Buffers;62 FunctionCallTrie::Allocators Allocators;63 FunctionCallTrie FCT;64 ThreadID TId;65};66 67using ThreadDataArray = Array<ThreadData>;68using ThreadDataAllocator = ThreadDataArray::AllocatorType;69 70// We use a separate buffer queue for the backing store for the allocator used71// by the ThreadData array. This lets us host the buffers, allocators, and tries72// associated with a thread by moving the data into the array instead of73// attempting to copy the data to a separately backed set of tries.74alignas(BufferQueue) static std::byte BufferQueueStorage[sizeof(BufferQueue)];75static BufferQueue *BQ = nullptr;76static BufferQueue::Buffer Buffer;77alignas(ThreadDataAllocator) static std::byte78 ThreadDataAllocatorStorage[sizeof(ThreadDataAllocator)];79alignas(ThreadDataArray) static std::byte80 ThreadDataArrayStorage[sizeof(ThreadDataArray)];81 82static ThreadDataAllocator *TDAllocator = nullptr;83static ThreadDataArray *TDArray = nullptr;84 85using ProfileBufferArray = Array<ProfileBuffer>;86using ProfileBufferArrayAllocator = typename ProfileBufferArray::AllocatorType;87 88// These need to be global aligned storage to avoid dynamic initialization. We89// need these to be aligned to allow us to placement new objects into the90// storage, and have pointers to those objects be appropriately aligned.91alignas(ProfileBufferArray) static std::byte92 ProfileBuffersStorage[sizeof(ProfileBufferArray)];93alignas(ProfileBufferArrayAllocator) static std::byte94 ProfileBufferArrayAllocatorStorage[sizeof(ProfileBufferArrayAllocator)];95 96static ProfileBufferArrayAllocator *ProfileBuffersAllocator = nullptr;97static ProfileBufferArray *ProfileBuffers = nullptr;98 99// Use a global flag to determine whether the collector implementation has been100// initialized.101static atomic_uint8_t CollectorInitialized{0};102 103} // namespace104 105void post(BufferQueue *Q, FunctionCallTrie &&T,106 FunctionCallTrie::Allocators &&A,107 FunctionCallTrie::Allocators::Buffers &&B,108 ThreadID TId) XRAY_NEVER_INSTRUMENT {109 DCHECK_NE(Q, nullptr);110 111 // Bail out early if the collector has not been initialized.112 if (!atomic_load(&CollectorInitialized, memory_order_acquire)) {113 T.~FunctionCallTrie();114 A.~Allocators();115 Q->releaseBuffer(B.NodeBuffer);116 Q->releaseBuffer(B.RootsBuffer);117 Q->releaseBuffer(B.ShadowStackBuffer);118 Q->releaseBuffer(B.NodeIdPairBuffer);119 B.~Buffers();120 return;121 }122 123 {124 SpinMutexLock Lock(&GlobalMutex);125 DCHECK_NE(TDAllocator, nullptr);126 DCHECK_NE(TDArray, nullptr);127 128 if (TDArray->AppendEmplace(Q, std::move(B), std::move(A), std::move(T),129 TId) == nullptr) {130 // If we fail to add the data to the array, we should destroy the objects131 // handed us.132 T.~FunctionCallTrie();133 A.~Allocators();134 Q->releaseBuffer(B.NodeBuffer);135 Q->releaseBuffer(B.RootsBuffer);136 Q->releaseBuffer(B.ShadowStackBuffer);137 Q->releaseBuffer(B.NodeIdPairBuffer);138 B.~Buffers();139 }140 }141}142 143// A PathArray represents the function id's representing a stack trace. In this144// context a path is almost always represented from the leaf function in a call145// stack to a root of the call trie.146using PathArray = Array<int32_t>;147 148struct ProfileRecord {149 using PathAllocator = typename PathArray::AllocatorType;150 151 // The Path in this record is the function id's from the leaf to the root of152 // the function call stack as represented from a FunctionCallTrie.153 PathArray Path;154 const FunctionCallTrie::Node *Node;155};156 157namespace {158 159using ProfileRecordArray = Array<ProfileRecord>;160 161// Walk a depth-first traversal of each root of the FunctionCallTrie to generate162// the path(s) and the data associated with the path.163static void164populateRecords(ProfileRecordArray &PRs, ProfileRecord::PathAllocator &PA,165 const FunctionCallTrie &Trie) XRAY_NEVER_INSTRUMENT {166 using StackArray = Array<const FunctionCallTrie::Node *>;167 using StackAllocator = typename StackArray::AllocatorType;168 StackAllocator StackAlloc(profilingFlags()->stack_allocator_max);169 StackArray DFSStack(StackAlloc);170 for (const auto *R : Trie.getRoots()) {171 DFSStack.Append(R);172 while (!DFSStack.empty()) {173 auto *Node = DFSStack.back();174 DFSStack.trim(1);175 if (Node == nullptr)176 continue;177 auto Record = PRs.AppendEmplace(PathArray{PA}, Node);178 if (Record == nullptr)179 return;180 DCHECK_NE(Record, nullptr);181 182 // Traverse the Node's parents and as we're doing so, get the FIds in183 // the order they appear.184 for (auto N = Node; N != nullptr; N = N->Parent)185 Record->Path.Append(N->FId);186 DCHECK(!Record->Path.empty());187 188 for (const auto C : Node->Callees)189 DFSStack.Append(C.NodePtr);190 }191 }192}193 194static void serializeRecords(ProfileBuffer *Buffer, const BlockHeader &Header,195 const ProfileRecordArray &ProfileRecords)196 XRAY_NEVER_INSTRUMENT {197 auto NextPtr = static_cast<uint8_t *>(198 internal_memcpy(Buffer->Data, &Header, sizeof(Header))) +199 sizeof(Header);200 for (const auto &Record : ProfileRecords) {201 // List of IDs follow:202 for (const auto FId : Record.Path)203 NextPtr =204 static_cast<uint8_t *>(internal_memcpy(NextPtr, &FId, sizeof(FId))) +205 sizeof(FId);206 207 // Add the sentinel here.208 constexpr int32_t SentinelFId = 0;209 NextPtr = static_cast<uint8_t *>(210 internal_memset(NextPtr, SentinelFId, sizeof(SentinelFId))) +211 sizeof(SentinelFId);212 213 // Add the node data here.214 NextPtr =215 static_cast<uint8_t *>(internal_memcpy(216 NextPtr, &Record.Node->CallCount, sizeof(Record.Node->CallCount))) +217 sizeof(Record.Node->CallCount);218 NextPtr = static_cast<uint8_t *>(219 internal_memcpy(NextPtr, &Record.Node->CumulativeLocalTime,220 sizeof(Record.Node->CumulativeLocalTime))) +221 sizeof(Record.Node->CumulativeLocalTime);222 }223 224 DCHECK_EQ(NextPtr - static_cast<uint8_t *>(Buffer->Data), Buffer->Size);225}226 227} // namespace228 229void serialize() XRAY_NEVER_INSTRUMENT {230 if (!atomic_load(&CollectorInitialized, memory_order_acquire))231 return;232 233 SpinMutexLock Lock(&GlobalMutex);234 235 // Clear out the global ProfileBuffers, if it's not empty.236 for (auto &B : *ProfileBuffers)237 deallocateBuffer(reinterpret_cast<unsigned char *>(B.Data), B.Size);238 ProfileBuffers->trim(ProfileBuffers->size());239 240 DCHECK_NE(TDArray, nullptr);241 if (TDArray->empty())242 return;243 244 // Then repopulate the global ProfileBuffers.245 u32 I = 0;246 auto MaxSize = profilingFlags()->global_allocator_max;247 auto ProfileArena = allocateBuffer(MaxSize);248 if (ProfileArena == nullptr)249 return;250 251 auto ProfileArenaCleanup = at_scope_exit(252 [&]() XRAY_NEVER_INSTRUMENT { deallocateBuffer(ProfileArena, MaxSize); });253 254 auto PathArena = allocateBuffer(profilingFlags()->global_allocator_max);255 if (PathArena == nullptr)256 return;257 258 auto PathArenaCleanup = at_scope_exit(259 [&]() XRAY_NEVER_INSTRUMENT { deallocateBuffer(PathArena, MaxSize); });260 261 for (const auto &ThreadTrie : *TDArray) {262 using ProfileRecordAllocator = typename ProfileRecordArray::AllocatorType;263 ProfileRecordAllocator PRAlloc(ProfileArena,264 profilingFlags()->global_allocator_max);265 ProfileRecord::PathAllocator PathAlloc(266 PathArena, profilingFlags()->global_allocator_max);267 ProfileRecordArray ProfileRecords(PRAlloc);268 269 // First, we want to compute the amount of space we're going to need. We'll270 // use a local allocator and an __xray::Array<...> to store the intermediary271 // data, then compute the size as we're going along. Then we'll allocate the272 // contiguous space to contain the thread buffer data.273 if (ThreadTrie.FCT.getRoots().empty())274 continue;275 276 populateRecords(ProfileRecords, PathAlloc, ThreadTrie.FCT);277 DCHECK(!ThreadTrie.FCT.getRoots().empty());278 DCHECK(!ProfileRecords.empty());279 280 // Go through each record, to compute the sizes.281 //282 // header size = block size (4 bytes)283 // + block number (4 bytes)284 // + thread id (8 bytes)285 // record size = path ids (4 bytes * number of ids + sentinel 4 bytes)286 // + call count (8 bytes)287 // + local time (8 bytes)288 // + end of record (8 bytes)289 u32 CumulativeSizes = 0;290 for (const auto &Record : ProfileRecords)291 CumulativeSizes += 20 + (4 * Record.Path.size());292 293 BlockHeader Header{16 + CumulativeSizes, I++, ThreadTrie.TId};294 auto B = ProfileBuffers->Append({});295 B->Size = sizeof(Header) + CumulativeSizes;296 B->Data = allocateBuffer(B->Size);297 DCHECK_NE(B->Data, nullptr);298 serializeRecords(B, Header, ProfileRecords);299 }300}301 302void reset() XRAY_NEVER_INSTRUMENT {303 atomic_store(&CollectorInitialized, 0, memory_order_release);304 SpinMutexLock Lock(&GlobalMutex);305 306 if (ProfileBuffers != nullptr) {307 // Clear out the profile buffers that have been serialized.308 for (auto &B : *ProfileBuffers)309 deallocateBuffer(reinterpret_cast<uint8_t *>(B.Data), B.Size);310 ProfileBuffers->trim(ProfileBuffers->size());311 ProfileBuffers = nullptr;312 }313 314 if (TDArray != nullptr) {315 // Release the resources as required.316 for (auto &TD : *TDArray) {317 TD.BQ->releaseBuffer(TD.Buffers.NodeBuffer);318 TD.BQ->releaseBuffer(TD.Buffers.RootsBuffer);319 TD.BQ->releaseBuffer(TD.Buffers.ShadowStackBuffer);320 TD.BQ->releaseBuffer(TD.Buffers.NodeIdPairBuffer);321 }322 // We don't bother destroying the array here because we've already323 // potentially freed the backing store for the array. Instead we're going to324 // reset the pointer to nullptr, and re-use the storage later instead325 // (placement-new'ing into the storage as-is).326 TDArray = nullptr;327 }328 329 if (TDAllocator != nullptr) {330 TDAllocator->~Allocator();331 TDAllocator = nullptr;332 }333 334 if (Buffer.Data != nullptr) {335 BQ->releaseBuffer(Buffer);336 }337 338 if (BQ == nullptr) {339 bool Success = false;340 new (&BufferQueueStorage)341 BufferQueue(profilingFlags()->global_allocator_max, 1, Success);342 if (!Success)343 return;344 BQ = reinterpret_cast<BufferQueue *>(&BufferQueueStorage);345 } else {346 BQ->finalize();347 348 if (BQ->init(profilingFlags()->global_allocator_max, 1) !=349 BufferQueue::ErrorCode::Ok)350 return;351 }352 353 if (BQ->getBuffer(Buffer) != BufferQueue::ErrorCode::Ok)354 return;355 356 new (&ProfileBufferArrayAllocatorStorage)357 ProfileBufferArrayAllocator(profilingFlags()->global_allocator_max);358 ProfileBuffersAllocator = reinterpret_cast<ProfileBufferArrayAllocator *>(359 &ProfileBufferArrayAllocatorStorage);360 361 new (&ProfileBuffersStorage) ProfileBufferArray(*ProfileBuffersAllocator);362 ProfileBuffers =363 reinterpret_cast<ProfileBufferArray *>(&ProfileBuffersStorage);364 365 new (&ThreadDataAllocatorStorage)366 ThreadDataAllocator(Buffer.Data, Buffer.Size);367 TDAllocator =368 reinterpret_cast<ThreadDataAllocator *>(&ThreadDataAllocatorStorage);369 new (&ThreadDataArrayStorage) ThreadDataArray(*TDAllocator);370 TDArray = reinterpret_cast<ThreadDataArray *>(&ThreadDataArrayStorage);371 372 atomic_store(&CollectorInitialized, 1, memory_order_release);373}374 375XRayBuffer nextBuffer(XRayBuffer B) XRAY_NEVER_INSTRUMENT {376 SpinMutexLock Lock(&GlobalMutex);377 378 if (ProfileBuffers == nullptr || ProfileBuffers->size() == 0)379 return {nullptr, 0};380 381 static pthread_once_t Once = PTHREAD_ONCE_INIT;382 alignas(XRayProfilingFileHeader) static std::byte383 FileHeaderStorage[sizeof(XRayProfilingFileHeader)];384 pthread_once(385 &Once, +[]() XRAY_NEVER_INSTRUMENT {386 new (&FileHeaderStorage) XRayProfilingFileHeader{};387 });388 389 if (UNLIKELY(B.Data == nullptr)) {390 // The first buffer should always contain the file header information.391 auto &FileHeader =392 *reinterpret_cast<XRayProfilingFileHeader *>(&FileHeaderStorage);393 FileHeader.Timestamp = NanoTime();394 FileHeader.PID = internal_getpid();395 return {&FileHeaderStorage, sizeof(XRayProfilingFileHeader)};396 }397 398 if (UNLIKELY(B.Data == &FileHeaderStorage))399 return {(*ProfileBuffers)[0].Data, (*ProfileBuffers)[0].Size};400 401 BlockHeader Header;402 internal_memcpy(&Header, B.Data, sizeof(BlockHeader));403 auto NextBlock = Header.BlockNum + 1;404 if (NextBlock < ProfileBuffers->size())405 return {(*ProfileBuffers)[NextBlock].Data,406 (*ProfileBuffers)[NextBlock].Size};407 return {nullptr, 0};408}409 410} // namespace profileCollectorService411} // namespace __xray412