270 lines · c
1//===-- PerThreadTable.h -- PerThread Storage Structure ----*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// Table indexed with one entry per thread.10//11//===----------------------------------------------------------------------===//12 13#ifndef OFFLOAD_PERTHREADTABLE_H14#define OFFLOAD_PERTHREADTABLE_H15 16#include <list>17#include <llvm/ADT/SmallVector.h>18#include <llvm/Support/Error.h>19#include <memory>20#include <mutex>21#include <type_traits>22 23template <typename ObjectType> class PerThread {24 std::mutex Mutex;25 llvm::SmallVector<std::shared_ptr<ObjectType>> ThreadDataList;26 27 ObjectType &getThreadData() {28 static thread_local std::shared_ptr<ObjectType> ThreadData = nullptr;29 if (!ThreadData) {30 ThreadData = std::make_shared<ObjectType>();31 std::lock_guard<std::mutex> Lock(Mutex);32 ThreadDataList.push_back(ThreadData);33 }34 return *ThreadData;35 }36 37public:38 // Define default constructors, disable copy and move constructors.39 PerThread() = default;40 PerThread(const PerThread &) = delete;41 PerThread(PerThread &&) = delete;42 PerThread &operator=(const PerThread &) = delete;43 PerThread &operator=(PerThread &&) = delete;44 ~PerThread() {45 assert(Mutex.try_lock() && (Mutex.unlock(), true) &&46 "Cannot be deleted while other threads are adding entries");47 ThreadDataList.clear();48 }49 50 ObjectType &get() { return getThreadData(); }51 52 template <class ClearFuncTy> void clear(ClearFuncTy ClearFunc) {53 assert(Mutex.try_lock() && (Mutex.unlock(), true) &&54 "Clear cannot be called while other threads are adding entries");55 for (std::shared_ptr<ObjectType> ThreadData : ThreadDataList) {56 if (!ThreadData)57 continue;58 ClearFunc(*ThreadData);59 }60 ThreadDataList.clear();61 }62};63 64template <typename ContainerTy> struct ContainerConcepts {65 template <typename, template <typename> class, typename = std::void_t<>>66 struct has : std::false_type {};67 template <typename Ty, template <typename> class Op>68 struct has<Ty, Op, std::void_t<Op<Ty>>> : std::true_type {};69 70 template <typename Ty> using IteratorTypeCheck = typename Ty::iterator;71 template <typename Ty> using MappedTypeCheck = typename Ty::mapped_type;72 template <typename Ty> using ValueTypeCheck = typename Ty::value_type;73 template <typename Ty> using KeyTypeCheck = typename Ty::key_type;74 template <typename Ty> using SizeTypeCheck = typename Ty::size_type;75 76 template <typename Ty>77 using ClearCheck = decltype(std::declval<Ty>().clear());78 template <typename Ty>79 using ReserveCheck = decltype(std::declval<Ty>().reserve(1));80 template <typename Ty>81 using ResizeCheck = decltype(std::declval<Ty>().resize(1));82 83 static constexpr bool hasIterator =84 has<ContainerTy, IteratorTypeCheck>::value;85 static constexpr bool hasClear = has<ContainerTy, ClearCheck>::value;86 static constexpr bool isAssociative =87 has<ContainerTy, MappedTypeCheck>::value;88 static constexpr bool hasReserve = has<ContainerTy, ReserveCheck>::value;89 static constexpr bool hasResize = has<ContainerTy, ResizeCheck>::value;90 91 template <typename, template <typename> class, typename = std::void_t<>>92 struct has_type {93 using type = void;94 };95 template <typename Ty, template <typename> class Op>96 struct has_type<Ty, Op, std::void_t<Op<Ty>>> {97 using type = Op<Ty>;98 };99 100 using iterator = typename has_type<ContainerTy, IteratorTypeCheck>::type;101 using value_type = typename std::conditional_t<102 isAssociative, typename has_type<ContainerTy, MappedTypeCheck>::type,103 typename has_type<ContainerTy, ValueTypeCheck>::type>;104 using key_type = typename std::conditional_t<105 isAssociative, typename has_type<ContainerTy, KeyTypeCheck>::type,106 typename has_type<ContainerTy, SizeTypeCheck>::type>;107};108 109// Using an STL container (such as std::vector) indexed by thread ID has110// too many race conditions issues so we store each thread entry into a111// thread_local variable.112// ContainerType is the container type used to store the objects, e.g.,113// std::vector, std::set, etc. by each thread. ObjectType is the type of the114// stored objects e.g., omp_interop_val_t *, ...115template <typename ContainerType, typename ObjectType> class PerThreadTable {116 using iterator = typename ContainerConcepts<ContainerType>::iterator;117 118 struct PerThreadData {119 size_t Size = 0;120 std::unique_ptr<ContainerType> ThreadEntry;121 };122 123 std::mutex Mutex;124 llvm::SmallVector<std::shared_ptr<PerThreadData>> ThreadDataList;125 126 PerThreadData &getThreadData() {127 static thread_local std::shared_ptr<PerThreadData> ThreadData = nullptr;128 if (!ThreadData) {129 ThreadData = std::make_shared<PerThreadData>();130 std::lock_guard<std::mutex> Lock(Mutex);131 ThreadDataList.push_back(ThreadData);132 }133 return *ThreadData;134 }135 136protected:137 ContainerType &getThreadEntry() {138 PerThreadData &ThreadData = getThreadData();139 if (ThreadData.ThreadEntry)140 return *ThreadData.ThreadEntry;141 ThreadData.ThreadEntry = std::make_unique<ContainerType>();142 return *ThreadData.ThreadEntry;143 }144 145 size_t &getThreadSize() {146 PerThreadData &ThreadData = getThreadData();147 return ThreadData.Size;148 }149 150 void setSize(size_t Size) {151 size_t &SizeRef = getThreadSize();152 SizeRef = Size;153 }154 155public:156 // define default constructors, disable copy and move constructors.157 PerThreadTable() = default;158 PerThreadTable(const PerThreadTable &) = delete;159 PerThreadTable(PerThreadTable &&) = delete;160 PerThreadTable &operator=(const PerThreadTable &) = delete;161 PerThreadTable &operator=(PerThreadTable &&) = delete;162 ~PerThreadTable() {163 assert(Mutex.try_lock() && (Mutex.unlock(), true) &&164 "Cannot be deleted while other threads are adding entries");165 ThreadDataList.clear();166 }167 168 void add(ObjectType obj) {169 ContainerType &Entry = getThreadEntry();170 size_t &SizeRef = getThreadSize();171 SizeRef++;172 Entry.add(obj);173 }174 175 iterator erase(iterator it) {176 ContainerType &Entry = getThreadEntry();177 size_t &SizeRef = getThreadSize();178 SizeRef--;179 return Entry.erase(it);180 }181 182 size_t size() { return getThreadSize(); }183 184 // Iterators to traverse objects owned by185 // the current thread.186 iterator begin() {187 ContainerType &Entry = getThreadEntry();188 return Entry.begin();189 }190 iterator end() {191 ContainerType &Entry = getThreadEntry();192 return Entry.end();193 }194 195 template <class ClearFuncTy> void clear(ClearFuncTy ClearFunc) {196 assert(Mutex.try_lock() && (Mutex.unlock(), true) &&197 "Clear cannot be called while other threads are adding entries");198 for (std::shared_ptr<PerThreadData> ThreadData : ThreadDataList) {199 if (!ThreadData->ThreadEntry || ThreadData->Size == 0)200 continue;201 if constexpr (ContainerConcepts<ContainerType>::hasIterator &&202 ContainerConcepts<ContainerType>::hasClear) {203 for (auto &Obj : *ThreadData->ThreadEntry) {204 if constexpr (ContainerConcepts<ContainerType>::isAssociative) {205 ClearFunc(Obj.second);206 } else {207 ClearFunc(Obj);208 }209 }210 ThreadData->ThreadEntry->clear();211 } else {212 static_assert(true, "Container type not supported");213 }214 ThreadData->Size = 0;215 }216 ThreadDataList.clear();217 }218 219 template <class DeinitFuncTy> llvm::Error deinit(DeinitFuncTy DeinitFunc) {220 assert(Mutex.try_lock() && (Mutex.unlock(), true) &&221 "Deinit cannot be called while other threads are adding entries");222 for (std::shared_ptr<PerThreadData> ThreadData : ThreadDataList) {223 if (!ThreadData->ThreadEntry || ThreadData->Size == 0)224 continue;225 for (auto &Obj : *ThreadData->ThreadEntry) {226 if constexpr (ContainerConcepts<ContainerType>::isAssociative) {227 if (auto Err = DeinitFunc(Obj.second))228 return Err;229 } else {230 if (auto Err = DeinitFunc(Obj))231 return Err;232 }233 }234 }235 return llvm::Error::success();236 }237};238 239template <typename ContainerType, size_t ReserveSize = 0>240class PerThreadContainer241 : public PerThreadTable<ContainerType, typename ContainerConcepts<242 ContainerType>::value_type> {243 244 using IndexType = typename ContainerConcepts<ContainerType>::key_type;245 using ObjectType = typename ContainerConcepts<ContainerType>::value_type;246 247public:248 // Get the object for the given index in the current thread.249 ObjectType &get(IndexType Index) {250 ContainerType &Entry = this->getThreadEntry();251 252 // Specialized code for vector-like containers.253 if constexpr (ContainerConcepts<ContainerType>::hasResize) {254 if (Index >= Entry.size()) {255 if constexpr (ContainerConcepts<ContainerType>::hasReserve &&256 ReserveSize > 0)257 Entry.reserve(ReserveSize);258 259 // If the index is out of bounds, try resize the container.260 Entry.resize(Index + 1);261 }262 }263 ObjectType &Ret = Entry[Index];264 this->setSize(Entry.size());265 return Ret;266 }267};268 269#endif270