393 lines · c
1// -*- C++ -*-2//===----------------------------------------------------------------------===//3//4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.5// See https://llvm.org/LICENSE.txt for license information.6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception7//8// Kokkos v. 4.09// Copyright (2022) National Technology & Engineering10// Solutions of Sandia, LLC (NTESS).11//12// Under the terms of Contract DE-NA0003525 with NTESS,13// the U.S. Government retains certain rights in this software.14//15//===---------------------------------------------------------------------===//16 17#ifndef _LIBCPP___ATOMIC_ATOMIC_REF_H18#define _LIBCPP___ATOMIC_ATOMIC_REF_H19 20#include <__assert>21#include <__atomic/atomic_sync.h>22#include <__atomic/check_memory_order.h>23#include <__atomic/floating_point_helper.h>24#include <__atomic/memory_order.h>25#include <__atomic/to_gcc_order.h>26#include <__concepts/arithmetic.h>27#include <__concepts/same_as.h>28#include <__config>29#include <__cstddef/byte.h>30#include <__cstddef/ptrdiff_t.h>31#include <__memory/addressof.h>32#include <__type_traits/has_unique_object_representation.h>33#include <__type_traits/is_trivially_copyable.h>34#include <cstdint>35#include <cstring>36 37#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)38# pragma GCC system_header39#endif40 41_LIBCPP_PUSH_MACROS42#include <__undef_macros>43 44_LIBCPP_BEGIN_NAMESPACE_STD45 46#if _LIBCPP_STD_VER >= 2047 48// These types are required to make __atomic_is_always_lock_free work across GCC and Clang.49// The purpose of this trick is to make sure that we provide an object with the correct alignment50// to __atomic_is_always_lock_free, since that answer depends on the alignment.51template <size_t _Alignment>52struct __alignment_checker_type {53 alignas(_Alignment) char __data;54};55 56template <size_t _Alignment>57struct __get_aligner_instance {58 static constexpr __alignment_checker_type<_Alignment> __instance{};59};60 61template <class _Tp>62struct __atomic_ref_base {63private:64 _LIBCPP_HIDE_FROM_ABI static _Tp* __clear_padding(_Tp& __val) noexcept {65 _Tp* __ptr = std::addressof(__val);66# if __has_builtin(__builtin_clear_padding)67 __builtin_clear_padding(__ptr);68# endif69 return __ptr;70 }71 72 _LIBCPP_HIDE_FROM_ABI static bool __compare_exchange(73 _Tp* __ptr, _Tp* __expected, _Tp* __desired, bool __is_weak, int __success, int __failure) noexcept {74 if constexpr (75# if __has_builtin(__builtin_clear_padding)76 has_unique_object_representations_v<_Tp> || floating_point<_Tp>77# else78 true // NOLINT(readability-simplify-boolean-expr)79# endif80 ) {81 return __atomic_compare_exchange(__ptr, __expected, __desired, __is_weak, __success, __failure);82 } else { // _Tp has padding bits and __builtin_clear_padding is available83 __clear_padding(*__desired);84 _Tp __copy = *__expected;85 __clear_padding(__copy);86 // The algorithm we use here is basically to perform `__atomic_compare_exchange` on the87 // values until it has either succeeded, or failed because the value representation of the88 // objects involved was different. This is why we loop around __atomic_compare_exchange:89 // we basically loop until its failure is caused by the value representation of the objects90 // being different, not only their object representation.91 while (true) {92 _Tp __prev = __copy;93 if (__atomic_compare_exchange(__ptr, std::addressof(__copy), __desired, __is_weak, __success, __failure)) {94 return true;95 }96 _Tp __curr = __copy;97 if (std::memcmp(__clear_padding(__prev), __clear_padding(__curr), sizeof(_Tp)) != 0) {98 // Value representation without padding bits do not compare equal ->99 // write the current content of *ptr into *expected100 std::memcpy(__expected, std::addressof(__copy), sizeof(_Tp));101 return false;102 }103 }104 }105 }106 107 friend struct __atomic_waitable_traits<__atomic_ref_base<_Tp>>;108 109 // require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned to at least their size to be potentially110 // used lock-free111 static constexpr size_t __min_alignment = (sizeof(_Tp) & (sizeof(_Tp) - 1)) || (sizeof(_Tp) > 16) ? 0 : sizeof(_Tp);112 113public:114 using value_type = _Tp;115 116 static constexpr size_t required_alignment = alignof(_Tp) > __min_alignment ? alignof(_Tp) : __min_alignment;117 118 // The __atomic_always_lock_free builtin takes into account the alignment of the pointer if provided,119 // so we create a fake pointer with a suitable alignment when querying it. Note that we are guaranteed120 // that the pointer is going to be aligned properly at runtime because that is a (checked) precondition121 // of atomic_ref's constructor.122 static constexpr bool is_always_lock_free =123 __atomic_always_lock_free(sizeof(_Tp), std::addressof(__get_aligner_instance<required_alignment>::__instance));124 125 _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const noexcept { return __atomic_is_lock_free(sizeof(_Tp), __ptr_); }126 127 _LIBCPP_HIDE_FROM_ABI void store(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept128 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__order) {129 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(130 __order == memory_order::relaxed || __order == memory_order::release || __order == memory_order::seq_cst,131 "atomic_ref: memory order argument to atomic store operation is invalid");132 __atomic_store(__ptr_, __clear_padding(__desired), std::__to_gcc_order(__order));133 }134 135 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept {136 store(__desired);137 return __desired;138 }139 140 _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __order = memory_order::seq_cst) const noexcept141 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__order) {142 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(143 __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||144 __order == memory_order::seq_cst,145 "atomic_ref: memory order argument to atomic load operation is invalid");146 alignas(_Tp) byte __mem[sizeof(_Tp)];147 auto* __ret = reinterpret_cast<_Tp*>(__mem);148 __atomic_load(__ptr_, __ret, std::__to_gcc_order(__order));149 return *__ret;150 }151 152 _LIBCPP_HIDE_FROM_ABI operator _Tp() const noexcept { return load(); }153 154 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {155 alignas(_Tp) byte __mem[sizeof(_Tp)];156 auto* __ret = reinterpret_cast<_Tp*>(__mem);157 __atomic_exchange(__ptr_, __clear_padding(__desired), __ret, std::__to_gcc_order(__order));158 return *__ret;159 }160 161 _LIBCPP_HIDE_FROM_ABI bool162 compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept163 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {164 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(165 __failure == memory_order::relaxed || __failure == memory_order::consume ||166 __failure == memory_order::acquire || __failure == memory_order::seq_cst,167 "atomic_ref: failure memory order argument to weak atomic compare-and-exchange operation is invalid");168 return __compare_exchange(169 __ptr_,170 std::addressof(__expected),171 std::addressof(__desired),172 true,173 std::__to_gcc_order(__success),174 std::__to_gcc_order(__failure));175 }176 _LIBCPP_HIDE_FROM_ABI bool177 compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __success, memory_order __failure) const noexcept178 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__success, __failure) {179 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(180 __failure == memory_order::relaxed || __failure == memory_order::consume ||181 __failure == memory_order::acquire || __failure == memory_order::seq_cst,182 "atomic_ref: failure memory order argument to strong atomic compare-and-exchange operation is invalid");183 return __compare_exchange(184 __ptr_,185 std::addressof(__expected),186 std::addressof(__desired),187 false,188 std::__to_gcc_order(__success),189 std::__to_gcc_order(__failure));190 }191 192 _LIBCPP_HIDE_FROM_ABI bool193 compare_exchange_weak(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {194 return __compare_exchange(195 __ptr_,196 std::addressof(__expected),197 std::addressof(__desired),198 true,199 std::__to_gcc_order(__order),200 std::__to_gcc_failure_order(__order));201 }202 _LIBCPP_HIDE_FROM_ABI bool203 compare_exchange_strong(_Tp& __expected, _Tp __desired, memory_order __order = memory_order::seq_cst) const noexcept {204 return __compare_exchange(205 __ptr_,206 std::addressof(__expected),207 std::addressof(__desired),208 false,209 std::__to_gcc_order(__order),210 std::__to_gcc_failure_order(__order));211 }212 213 _LIBCPP_HIDE_FROM_ABI void wait(_Tp __old, memory_order __order = memory_order::seq_cst) const noexcept214 _LIBCPP_CHECK_WAIT_MEMORY_ORDER(__order) {215 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(216 __order == memory_order::relaxed || __order == memory_order::consume || __order == memory_order::acquire ||217 __order == memory_order::seq_cst,218 "atomic_ref: memory order argument to atomic wait operation is invalid");219 std::__atomic_wait(*this, __old, __order);220 }221 _LIBCPP_HIDE_FROM_ABI void notify_one() const noexcept { std::__atomic_notify_one(*this); }222 _LIBCPP_HIDE_FROM_ABI void notify_all() const noexcept { std::__atomic_notify_all(*this); }223# if _LIBCPP_STD_VER >= 26224 [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr _Tp* address() const noexcept { return __ptr_; }225# endif226 227protected:228 using _Aligned_Tp [[__gnu__::__aligned__(required_alignment), __gnu__::__nodebug__]] = _Tp;229 _Aligned_Tp* __ptr_;230 231 _LIBCPP_HIDE_FROM_ABI __atomic_ref_base(_Tp& __obj) : __ptr_(std::addressof(__obj)) {}232};233 234template <class _Tp>235struct __atomic_waitable_traits<__atomic_ref_base<_Tp>> {236 static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_ref_base<_Tp>& __a, memory_order __order) {237 return __a.load(__order);238 }239 static _LIBCPP_HIDE_FROM_ABI const _Tp* __atomic_contention_address(const __atomic_ref_base<_Tp>& __a) {240 return __a.__ptr_;241 }242};243 244template <class _Tp>245struct atomic_ref : public __atomic_ref_base<_Tp> {246 static_assert(is_trivially_copyable_v<_Tp>, "std::atomic_ref<T> requires that 'T' be a trivially copyable type");247 248 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;249 250 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {251 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(252 reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0,253 "atomic_ref ctor: referenced object must be aligned to required_alignment");254 }255 256 _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;257 258 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }259 260 atomic_ref& operator=(const atomic_ref&) = delete;261};262 263template <class _Tp>264 requires(std::integral<_Tp> && !std::same_as<bool, _Tp>)265struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {266 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;267 268 using difference_type = __base::value_type;269 270 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {271 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(272 reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0,273 "atomic_ref ctor: referenced object must be aligned to required_alignment");274 }275 276 _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;277 278 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }279 280 atomic_ref& operator=(const atomic_ref&) = delete;281 282 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {283 return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));284 }285 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {286 return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order));287 }288 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {289 return __atomic_fetch_and(this->__ptr_, __arg, std::__to_gcc_order(__order));290 }291 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {292 return __atomic_fetch_or(this->__ptr_, __arg, std::__to_gcc_order(__order));293 }294 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {295 return __atomic_fetch_xor(this->__ptr_, __arg, std::__to_gcc_order(__order));296 }297 298 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) const noexcept { return fetch_add(_Tp(1)); }299 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) const noexcept { return fetch_sub(_Tp(1)); }300 _LIBCPP_HIDE_FROM_ABI _Tp operator++() const noexcept { return fetch_add(_Tp(1)) + _Tp(1); }301 _LIBCPP_HIDE_FROM_ABI _Tp operator--() const noexcept { return fetch_sub(_Tp(1)) - _Tp(1); }302 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }303 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }304 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __arg) const noexcept { return fetch_and(__arg) & __arg; }305 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __arg) const noexcept { return fetch_or(__arg) | __arg; }306 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __arg) const noexcept { return fetch_xor(__arg) ^ __arg; }307};308 309template <class _Tp>310 requires std::floating_point<_Tp>311struct atomic_ref<_Tp> : public __atomic_ref_base<_Tp> {312 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp>;313 314 using difference_type = __base::value_type;315 316 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp& __obj) : __base(__obj) {317 _LIBCPP_ASSERT_ARGUMENT_WITHIN_DOMAIN(318 reinterpret_cast<uintptr_t>(std::addressof(__obj)) % __base::required_alignment == 0,319 "atomic_ref ctor: referenced object must be aligned to required_alignment");320 }321 322 _LIBCPP_HIDE_FROM_ABI atomic_ref(const atomic_ref&) noexcept = default;323 324 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __desired) const noexcept { return __base::operator=(__desired); }325 326 atomic_ref& operator=(const atomic_ref&) = delete;327 328 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {329 if constexpr (std::__has_rmw_builtin<_Tp>()) {330 return __atomic_fetch_add(this->__ptr_, __arg, std::__to_gcc_order(__order));331 } else {332 _Tp __old = this->load(memory_order_relaxed);333 _Tp __new = __old + __arg;334 while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {335 __new = __old + __arg;336 }337 return __old;338 }339 }340 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __arg, memory_order __order = memory_order_seq_cst) const noexcept {341 if constexpr (std::__has_rmw_builtin<_Tp>()) {342 return __atomic_fetch_sub(this->__ptr_, __arg, std::__to_gcc_order(__order));343 } else {344 _Tp __old = this->load(memory_order_relaxed);345 _Tp __new = __old - __arg;346 while (!this->compare_exchange_weak(__old, __new, __order, memory_order_relaxed)) {347 __new = __old - __arg;348 }349 return __old;350 }351 }352 353 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __arg) const noexcept { return fetch_add(__arg) + __arg; }354 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __arg) const noexcept { return fetch_sub(__arg) - __arg; }355};356 357template <class _Tp>358struct atomic_ref<_Tp*> : public __atomic_ref_base<_Tp*> {359 using __base _LIBCPP_NODEBUG = __atomic_ref_base<_Tp*>;360 361 using difference_type = ptrdiff_t;362 363 _LIBCPP_HIDE_FROM_ABI explicit atomic_ref(_Tp*& __ptr) : __base(__ptr) {}364 365 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __desired) const noexcept { return __base::operator=(__desired); }366 367 atomic_ref& operator=(const atomic_ref&) = delete;368 369 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {370 return __atomic_fetch_add(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));371 }372 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __arg, memory_order __order = memory_order_seq_cst) const noexcept {373 return __atomic_fetch_sub(this->__ptr_, __arg * sizeof(_Tp), std::__to_gcc_order(__order));374 }375 376 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) const noexcept { return fetch_add(1); }377 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) const noexcept { return fetch_sub(1); }378 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() const noexcept { return fetch_add(1) + 1; }379 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() const noexcept { return fetch_sub(1) - 1; }380 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __arg) const noexcept { return fetch_add(__arg) + __arg; }381 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __arg) const noexcept { return fetch_sub(__arg) - __arg; }382};383 384_LIBCPP_CTAD_SUPPORTED_FOR_TYPE(atomic_ref);385 386#endif // _LIBCPP_STD_VER >= 20387 388_LIBCPP_END_NAMESPACE_STD389 390_LIBCPP_POP_MACROS391 392#endif // _LIBCPP__ATOMIC_ATOMIC_REF_H393