238 lines · cpp
1//===-- xray_buffer_queue.cpp ----------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a part of XRay, a dynamic runtime instrumentation system.10//11// Defines the interface for a buffer queue implementation.12//13//===----------------------------------------------------------------------===//14#include "xray_buffer_queue.h"15#include "sanitizer_common/sanitizer_atomic.h"16#include "sanitizer_common/sanitizer_common.h"17#include "sanitizer_common/sanitizer_libc.h"18#if !SANITIZER_FUCHSIA19#include "sanitizer_common/sanitizer_posix.h"20#endif21#include "xray_allocator.h"22#include "xray_defs.h"23#include <memory>24#include <sys/mman.h>25 26using namespace __xray;27 28namespace {29 30BufferQueue::ControlBlock *allocControlBlock(size_t Size, size_t Count) {31 auto B =32 allocateBuffer((sizeof(BufferQueue::ControlBlock) - 1) + (Size * Count));33 return B == nullptr ? nullptr34 : reinterpret_cast<BufferQueue::ControlBlock *>(B);35}36 37void deallocControlBlock(BufferQueue::ControlBlock *C, size_t Size,38 size_t Count) {39 deallocateBuffer(reinterpret_cast<unsigned char *>(C),40 (sizeof(BufferQueue::ControlBlock) - 1) + (Size * Count));41}42 43void decRefCount(BufferQueue::ControlBlock *C, size_t Size, size_t Count) {44 if (C == nullptr)45 return;46 if (atomic_fetch_sub(&C->RefCount, 1, memory_order_acq_rel) == 1)47 deallocControlBlock(C, Size, Count);48}49 50void incRefCount(BufferQueue::ControlBlock *C) {51 if (C == nullptr)52 return;53 atomic_fetch_add(&C->RefCount, 1, memory_order_acq_rel);54}55 56// We use a struct to ensure that we are allocating one atomic_uint64_t per57// cache line. This allows us to not worry about false-sharing among atomic58// objects being updated (constantly) by different threads.59struct ExtentsPadded {60 union {61 atomic_uint64_t Extents;62 unsigned char Storage[kCacheLineSize];63 };64};65 66constexpr size_t kExtentsSize = sizeof(ExtentsPadded);67 68} // namespace69 70BufferQueue::ErrorCode BufferQueue::init(size_t BS, size_t BC) {71 SpinMutexLock Guard(&Mutex);72 73 if (!finalizing())74 return BufferQueue::ErrorCode::AlreadyInitialized;75 76 cleanupBuffers();77 78 bool Success = false;79 BufferSize = BS;80 BufferCount = BC;81 82 BackingStore = allocControlBlock(BufferSize, BufferCount);83 if (BackingStore == nullptr)84 return BufferQueue::ErrorCode::NotEnoughMemory;85 86 auto CleanupBackingStore = at_scope_exit([&, this] {87 if (Success)88 return;89 deallocControlBlock(BackingStore, BufferSize, BufferCount);90 BackingStore = nullptr;91 });92 93 // Initialize enough atomic_uint64_t instances, each94 ExtentsBackingStore = allocControlBlock(kExtentsSize, BufferCount);95 if (ExtentsBackingStore == nullptr)96 return BufferQueue::ErrorCode::NotEnoughMemory;97 98 auto CleanupExtentsBackingStore = at_scope_exit([&, this] {99 if (Success)100 return;101 deallocControlBlock(ExtentsBackingStore, kExtentsSize, BufferCount);102 ExtentsBackingStore = nullptr;103 });104 105 Buffers = initArray<BufferRep>(BufferCount);106 if (Buffers == nullptr)107 return BufferQueue::ErrorCode::NotEnoughMemory;108 109 // At this point we increment the generation number to associate the buffers110 // to the new generation.111 atomic_fetch_add(&Generation, 1, memory_order_acq_rel);112 113 // First, we initialize the refcount in the ControlBlock, which we treat as114 // being at the start of the BackingStore pointer.115 atomic_store(&BackingStore->RefCount, 1, memory_order_release);116 atomic_store(&ExtentsBackingStore->RefCount, 1, memory_order_release);117 118 // Then we initialise the individual buffers that sub-divide the whole backing119 // store. Each buffer will start at the `Data` member of the ControlBlock, and120 // will be offsets from these locations.121 for (size_t i = 0; i < BufferCount; ++i) {122 auto &T = Buffers[i];123 auto &Buf = T.Buff;124 auto *E = reinterpret_cast<ExtentsPadded *>(&ExtentsBackingStore->Data +125 (kExtentsSize * i));126 Buf.Extents = &E->Extents;127 atomic_store(Buf.Extents, 0, memory_order_release);128 Buf.Generation = generation();129 Buf.Data = &BackingStore->Data + (BufferSize * i);130 Buf.Size = BufferSize;131 Buf.BackingStore = BackingStore;132 Buf.ExtentsBackingStore = ExtentsBackingStore;133 Buf.Count = BufferCount;134 T.Used = false;135 }136 137 Next = Buffers;138 First = Buffers;139 LiveBuffers = 0;140 atomic_store(&Finalizing, 0, memory_order_release);141 Success = true;142 return BufferQueue::ErrorCode::Ok;143}144 145BufferQueue::BufferQueue(size_t B, size_t N,146 bool &Success) XRAY_NEVER_INSTRUMENT147 : BufferSize(B),148 BufferCount(N),149 Mutex(),150 Finalizing{1},151 BackingStore(nullptr),152 ExtentsBackingStore(nullptr),153 Buffers(nullptr),154 Next(Buffers),155 First(Buffers),156 LiveBuffers(0),157 Generation{0} {158 Success = init(B, N) == BufferQueue::ErrorCode::Ok;159}160 161BufferQueue::ErrorCode BufferQueue::getBuffer(Buffer &Buf) {162 if (atomic_load(&Finalizing, memory_order_acquire))163 return ErrorCode::QueueFinalizing;164 165 BufferRep *B = nullptr;166 {167 SpinMutexLock Guard(&Mutex);168 if (LiveBuffers == BufferCount)169 return ErrorCode::NotEnoughMemory;170 B = Next++;171 if (Next == (Buffers + BufferCount))172 Next = Buffers;173 ++LiveBuffers;174 }175 176 incRefCount(BackingStore);177 incRefCount(ExtentsBackingStore);178 Buf = B->Buff;179 Buf.Generation = generation();180 B->Used = true;181 return ErrorCode::Ok;182}183 184BufferQueue::ErrorCode BufferQueue::releaseBuffer(Buffer &Buf) {185 // Check whether the buffer being referred to is within the bounds of the186 // backing store's range.187 BufferRep *B = nullptr;188 {189 SpinMutexLock Guard(&Mutex);190 if (Buf.Generation != generation() || LiveBuffers == 0) {191 Buf = {};192 decRefCount(Buf.BackingStore, Buf.Size, Buf.Count);193 decRefCount(Buf.ExtentsBackingStore, kExtentsSize, Buf.Count);194 return BufferQueue::ErrorCode::Ok;195 }196 197 if (Buf.Data < &BackingStore->Data ||198 Buf.Data > &BackingStore->Data + (BufferCount * BufferSize))199 return BufferQueue::ErrorCode::UnrecognizedBuffer;200 201 --LiveBuffers;202 B = First++;203 if (First == (Buffers + BufferCount))204 First = Buffers;205 }206 207 // Now that the buffer has been released, we mark it as "used".208 B->Buff = Buf;209 B->Used = true;210 decRefCount(Buf.BackingStore, Buf.Size, Buf.Count);211 decRefCount(Buf.ExtentsBackingStore, kExtentsSize, Buf.Count);212 atomic_store(B->Buff.Extents, atomic_load(Buf.Extents, memory_order_acquire),213 memory_order_release);214 Buf = {};215 return ErrorCode::Ok;216}217 218BufferQueue::ErrorCode BufferQueue::finalize() {219 if (atomic_exchange(&Finalizing, 1, memory_order_acq_rel))220 return ErrorCode::QueueFinalizing;221 return ErrorCode::Ok;222}223 224void BufferQueue::cleanupBuffers() {225 for (auto B = Buffers, E = Buffers + BufferCount; B != E; ++B)226 B->~BufferRep();227 deallocateBuffer(Buffers, BufferCount);228 decRefCount(BackingStore, BufferSize, BufferCount);229 decRefCount(ExtentsBackingStore, kExtentsSize, BufferCount);230 BackingStore = nullptr;231 ExtentsBackingStore = nullptr;232 Buffers = nullptr;233 BufferCount = 0;234 BufferSize = 0;235}236 237BufferQueue::~BufferQueue() { cleanupBuffers(); }238