801 lines · c
1//===----------------------------------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#ifndef _LIBCPP___ATOMIC_ATOMIC_H10#define _LIBCPP___ATOMIC_ATOMIC_H11 12#include <__atomic/atomic_sync.h>13#include <__atomic/check_memory_order.h>14#include <__atomic/floating_point_helper.h>15#include <__atomic/is_always_lock_free.h>16#include <__atomic/memory_order.h>17#include <__atomic/support.h>18#include <__config>19#include <__cstddef/ptrdiff_t.h>20#include <__memory/addressof.h>21#include <__type_traits/enable_if.h>22#include <__type_traits/is_floating_point.h>23#include <__type_traits/is_function.h>24#include <__type_traits/is_integral.h>25#include <__type_traits/is_nothrow_constructible.h>26#include <__type_traits/is_same.h>27#include <__type_traits/is_trivially_copyable.h>28#include <__type_traits/remove_const.h>29#include <__type_traits/remove_pointer.h>30#include <__type_traits/remove_volatile.h>31#include <__utility/forward.h>32#include <cstring>33 34#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)35# pragma GCC system_header36#endif37 38_LIBCPP_BEGIN_NAMESPACE_STD39 40template <class _Tp, bool = is_integral<_Tp>::value && !is_same<_Tp, bool>::value>41struct __atomic_base // false42{43 mutable __cxx_atomic_impl<_Tp> __a_;44 45 using value_type = _Tp;46 47#if _LIBCPP_STD_VER >= 1748 static constexpr bool is_always_lock_free = __libcpp_is_always_lock_free<__cxx_atomic_impl<_Tp> >::__value;49#endif50 51 _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const volatile _NOEXCEPT {52 return __cxx_atomic_is_lock_free(sizeof(__cxx_atomic_impl<_Tp>));53 }54 _LIBCPP_HIDE_FROM_ABI bool is_lock_free() const _NOEXCEPT {55 return static_cast<__atomic_base const volatile*>(this)->is_lock_free();56 }57 _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT58 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {59 std::__cxx_atomic_store(std::addressof(__a_), __d, __m);60 }61 _LIBCPP_HIDE_FROM_ABI void store(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT62 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {63 std::__cxx_atomic_store(std::addressof(__a_), __d, __m);64 }65 _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT66 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {67 return std::__cxx_atomic_load(std::addressof(__a_), __m);68 }69 _LIBCPP_HIDE_FROM_ABI _Tp load(memory_order __m = memory_order_seq_cst) const _NOEXCEPT70 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {71 return std::__cxx_atomic_load(std::addressof(__a_), __m);72 }73 _LIBCPP_HIDE_FROM_ABI operator _Tp() const volatile _NOEXCEPT { return load(); }74 _LIBCPP_HIDE_FROM_ABI operator _Tp() const _NOEXCEPT { return load(); }75 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {76 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);77 }78 _LIBCPP_HIDE_FROM_ABI _Tp exchange(_Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {79 return std::__cxx_atomic_exchange(std::addressof(__a_), __d, __m);80 }81 _LIBCPP_HIDE_FROM_ABI bool82 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT83 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {84 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);85 }86 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT87 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {88 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __s, __f);89 }90 _LIBCPP_HIDE_FROM_ABI bool91 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) volatile _NOEXCEPT92 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {93 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);94 }95 _LIBCPP_HIDE_FROM_ABI bool compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __s, memory_order __f) _NOEXCEPT96 _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {97 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __s, __f);98 }99 _LIBCPP_HIDE_FROM_ABI bool100 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {101 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);102 }103 _LIBCPP_HIDE_FROM_ABI bool104 compare_exchange_weak(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {105 return std::__cxx_atomic_compare_exchange_weak(std::addressof(__a_), std::addressof(__e), __d, __m, __m);106 }107 _LIBCPP_HIDE_FROM_ABI bool108 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {109 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);110 }111 _LIBCPP_HIDE_FROM_ABI bool112 compare_exchange_strong(_Tp& __e, _Tp __d, memory_order __m = memory_order_seq_cst) _NOEXCEPT {113 return std::__cxx_atomic_compare_exchange_strong(std::addressof(__a_), std::addressof(__e), __d, __m, __m);114 }115 116#if _LIBCPP_STD_VER >= 20117 _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT {118 std::__atomic_wait(*this, __v, __m);119 }120 _LIBCPP_HIDE_FROM_ABI void wait(_Tp __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {121 std::__atomic_wait(*this, __v, __m);122 }123 _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT { std::__atomic_notify_one(*this); }124 _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }125 _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT { std::__atomic_notify_all(*this); }126 _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }127#endif // _LIBCPP_STD_VER >= 20128 129#if _LIBCPP_STD_VER >= 20130 _LIBCPP_HIDE_FROM_ABI constexpr __atomic_base() noexcept(is_nothrow_default_constructible_v<_Tp>) : __a_(_Tp()) {}131#else132 _LIBCPP_HIDE_FROM_ABI __atomic_base() _NOEXCEPT = default;133#endif134 135 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __a_(__d) {}136 137 __atomic_base(const __atomic_base&) = delete;138};139 140// atomic<Integral>141 142template <class _Tp>143struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> {144 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp, false>;145 146 using difference_type = typename __base::value_type;147 148 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __atomic_base() _NOEXCEPT = default;149 150 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __atomic_base(_Tp __d) _NOEXCEPT : __base(__d) {}151 152 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {153 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);154 }155 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {156 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);157 }158 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {159 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);160 }161 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {162 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);163 }164 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {165 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);166 }167 _LIBCPP_HIDE_FROM_ABI _Tp fetch_and(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {168 return std::__cxx_atomic_fetch_and(std::addressof(this->__a_), __op, __m);169 }170 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {171 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);172 }173 _LIBCPP_HIDE_FROM_ABI _Tp fetch_or(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {174 return std::__cxx_atomic_fetch_or(std::addressof(this->__a_), __op, __m);175 }176 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {177 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);178 }179 _LIBCPP_HIDE_FROM_ABI _Tp fetch_xor(_Tp __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {180 return std::__cxx_atomic_fetch_xor(std::addressof(this->__a_), __op, __m);181 }182 183 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) volatile _NOEXCEPT { return fetch_add(_Tp(1)); }184 _LIBCPP_HIDE_FROM_ABI _Tp operator++(int) _NOEXCEPT { return fetch_add(_Tp(1)); }185 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) volatile _NOEXCEPT { return fetch_sub(_Tp(1)); }186 _LIBCPP_HIDE_FROM_ABI _Tp operator--(int) _NOEXCEPT { return fetch_sub(_Tp(1)); }187 _LIBCPP_HIDE_FROM_ABI _Tp operator++() volatile _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }188 _LIBCPP_HIDE_FROM_ABI _Tp operator++() _NOEXCEPT { return fetch_add(_Tp(1)) + _Tp(1); }189 _LIBCPP_HIDE_FROM_ABI _Tp operator--() volatile _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }190 _LIBCPP_HIDE_FROM_ABI _Tp operator--() _NOEXCEPT { return fetch_sub(_Tp(1)) - _Tp(1); }191 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }192 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) _NOEXCEPT { return fetch_add(__op) + __op; }193 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }194 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) _NOEXCEPT { return fetch_sub(__op) - __op; }195 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) volatile _NOEXCEPT { return fetch_and(__op) & __op; }196 _LIBCPP_HIDE_FROM_ABI _Tp operator&=(_Tp __op) _NOEXCEPT { return fetch_and(__op) & __op; }197 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) volatile _NOEXCEPT { return fetch_or(__op) | __op; }198 _LIBCPP_HIDE_FROM_ABI _Tp operator|=(_Tp __op) _NOEXCEPT { return fetch_or(__op) | __op; }199 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) volatile _NOEXCEPT { return fetch_xor(__op) ^ __op; }200 _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; }201};202 203// Here we need _IsIntegral because the default template argument is not enough204// e.g __atomic_base<int> is __atomic_base<int, true>, which inherits from205// __atomic_base<int, false> and the caller of the wait function is206// __atomic_base<int, false>. So specializing __atomic_base<_Tp> does not work207template <class _Tp, bool _IsIntegral>208struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > {209 static _LIBCPP_HIDE_FROM_ABI _Tp __atomic_load(const __atomic_base<_Tp, _IsIntegral>& __a, memory_order __order) {210 return __a.load(__order);211 }212 213 static _LIBCPP_HIDE_FROM_ABI _Tp214 __atomic_load(const volatile __atomic_base<_Tp, _IsIntegral>& __this, memory_order __order) {215 return __this.load(__order);216 }217 218 static _LIBCPP_HIDE_FROM_ABI const __cxx_atomic_impl<_Tp>*219 __atomic_contention_address(const __atomic_base<_Tp, _IsIntegral>& __a) {220 return std::addressof(__a.__a_);221 }222 223 static _LIBCPP_HIDE_FROM_ABI const volatile __cxx_atomic_impl<_Tp>*224 __atomic_contention_address(const volatile __atomic_base<_Tp, _IsIntegral>& __this) {225 return std::addressof(__this.__a_);226 }227};228 229template <typename _Tp>230struct __check_atomic_mandates {231 using type _LIBCPP_NODEBUG = _Tp;232 static_assert(is_trivially_copyable<_Tp>::value, "std::atomic<T> requires that 'T' be a trivially copyable type");233};234 235template <class _Tp>236struct atomic : public __atomic_base<typename __check_atomic_mandates<_Tp>::type> {237 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>;238 239#if _LIBCPP_STD_VER >= 20240 _LIBCPP_HIDE_FROM_ABI atomic() = default;241#else242 _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default;243#endif244 245 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp __d) _NOEXCEPT : __base(__d) {}246 247 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile _NOEXCEPT {248 __base::store(__d);249 return __d;250 }251 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) _NOEXCEPT {252 __base::store(__d);253 return __d;254 }255 256 atomic& operator=(const atomic&) = delete;257 atomic& operator=(const atomic&) volatile = delete;258};259 260// atomic<T*>261 262template <class _Tp>263struct atomic<_Tp*> : public __atomic_base<_Tp*> {264 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp*>;265 266 using difference_type = ptrdiff_t;267 268 _LIBCPP_HIDE_FROM_ABI atomic() _NOEXCEPT = default;269 270 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR atomic(_Tp* __d) _NOEXCEPT : __base(__d) {}271 272 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) volatile _NOEXCEPT {273 __base::store(__d);274 return __d;275 }276 _LIBCPP_HIDE_FROM_ABI _Tp* operator=(_Tp* __d) _NOEXCEPT {277 __base::store(__d);278 return __d;279 }280 281 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {282 // __atomic_fetch_add accepts function pointers, guard against them.283 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");284 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);285 }286 287 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_add(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {288 // __atomic_fetch_add accepts function pointers, guard against them.289 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");290 return std::__cxx_atomic_fetch_add(std::addressof(this->__a_), __op, __m);291 }292 293 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) volatile _NOEXCEPT {294 // __atomic_fetch_add accepts function pointers, guard against them.295 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");296 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);297 }298 299 _LIBCPP_HIDE_FROM_ABI _Tp* fetch_sub(ptrdiff_t __op, memory_order __m = memory_order_seq_cst) _NOEXCEPT {300 // __atomic_fetch_add accepts function pointers, guard against them.301 static_assert(!is_function<__remove_pointer_t<_Tp> >::value, "Pointer to function isn't allowed");302 return std::__cxx_atomic_fetch_sub(std::addressof(this->__a_), __op, __m);303 }304 305 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) volatile _NOEXCEPT { return fetch_add(1); }306 _LIBCPP_HIDE_FROM_ABI _Tp* operator++(int) _NOEXCEPT { return fetch_add(1); }307 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) volatile _NOEXCEPT { return fetch_sub(1); }308 _LIBCPP_HIDE_FROM_ABI _Tp* operator--(int) _NOEXCEPT { return fetch_sub(1); }309 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() volatile _NOEXCEPT { return fetch_add(1) + 1; }310 _LIBCPP_HIDE_FROM_ABI _Tp* operator++() _NOEXCEPT { return fetch_add(1) + 1; }311 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() volatile _NOEXCEPT { return fetch_sub(1) - 1; }312 _LIBCPP_HIDE_FROM_ABI _Tp* operator--() _NOEXCEPT { return fetch_sub(1) - 1; }313 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_add(__op) + __op; }314 _LIBCPP_HIDE_FROM_ABI _Tp* operator+=(ptrdiff_t __op) _NOEXCEPT { return fetch_add(__op) + __op; }315 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) volatile _NOEXCEPT { return fetch_sub(__op) - __op; }316 _LIBCPP_HIDE_FROM_ABI _Tp* operator-=(ptrdiff_t __op) _NOEXCEPT { return fetch_sub(__op) - __op; }317 318 atomic& operator=(const atomic&) = delete;319 atomic& operator=(const atomic&) volatile = delete;320};321 322template <class _Tp>323struct __atomic_waitable_traits<atomic<_Tp> > : __atomic_waitable_traits<__atomic_base<_Tp> > {};324 325#if _LIBCPP_STD_VER >= 20326template <class _Tp>327 requires is_floating_point_v<_Tp>328struct atomic<_Tp> : __atomic_base<_Tp> {329private:330 template <class _This, class _Operation, class _BuiltinOp>331 _LIBCPP_HIDE_FROM_ABI static _Tp332 __rmw_op(_This&& __self, _Tp __operand, memory_order __m, _Operation __operation, _BuiltinOp __builtin_op) {333 if constexpr (std::__has_rmw_builtin<_Tp>()) {334 return __builtin_op(std::addressof(std::forward<_This>(__self).__a_), __operand, __m);335 } else {336 _Tp __old = __self.load(memory_order_relaxed);337 _Tp __new = __operation(__old, __operand);338 while (!__self.compare_exchange_weak(__old, __new, __m, memory_order_relaxed)) {339# ifdef _LIBCPP_COMPILER_CLANG_BASED340 if constexpr (std::__is_fp80_long_double<_Tp>()) {341 // https://llvm.org/PR47978342 // clang bug: __old is not updated on failure for atomic<long double>::compare_exchange_weak343 // Note __old = __self.load(memory_order_relaxed) will not work344 std::__cxx_atomic_load_inplace(std::addressof(__self.__a_), std::addressof(__old), memory_order_relaxed);345 }346# endif347 __new = __operation(__old, __operand);348 }349 return __old;350 }351 }352 353 template <class _This>354 _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_add(_This&& __self, _Tp __operand, memory_order __m) {355 auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {356 return std::__cxx_atomic_fetch_add(__a, __builtin_operand, __order);357 };358 auto __plus = [](auto __a, auto __b) { return __a + __b; };359 return __rmw_op(std::forward<_This>(__self), __operand, __m, __plus, __builtin_op);360 }361 362 template <class _This>363 _LIBCPP_HIDE_FROM_ABI static _Tp __fetch_sub(_This&& __self, _Tp __operand, memory_order __m) {364 auto __builtin_op = [](auto __a, auto __builtin_operand, auto __order) {365 return std::__cxx_atomic_fetch_sub(__a, __builtin_operand, __order);366 };367 auto __minus = [](auto __a, auto __b) { return __a - __b; };368 return __rmw_op(std::forward<_This>(__self), __operand, __m, __minus, __builtin_op);369 }370 371public:372 using __base _LIBCPP_NODEBUG = __atomic_base<_Tp>;373 using value_type = _Tp;374 using difference_type = value_type;375 376 _LIBCPP_HIDE_FROM_ABI constexpr atomic() noexcept = default;377 _LIBCPP_HIDE_FROM_ABI constexpr atomic(_Tp __d) noexcept : __base(__d) {}378 379 atomic(const atomic&) = delete;380 atomic& operator=(const atomic&) = delete;381 atomic& operator=(const atomic&) volatile = delete;382 383 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) volatile noexcept384 requires __base::is_always_lock_free385 {386 __base::store(__d);387 return __d;388 }389 _LIBCPP_HIDE_FROM_ABI _Tp operator=(_Tp __d) noexcept {390 __base::store(__d);391 return __d;392 }393 394 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept395 requires __base::is_always_lock_free396 {397 return __fetch_add(*this, __op, __m);398 }399 400 _LIBCPP_HIDE_FROM_ABI _Tp fetch_add(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {401 return __fetch_add(*this, __op, __m);402 }403 404 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) volatile noexcept405 requires __base::is_always_lock_free406 {407 return __fetch_sub(*this, __op, __m);408 }409 410 _LIBCPP_HIDE_FROM_ABI _Tp fetch_sub(_Tp __op, memory_order __m = memory_order_seq_cst) noexcept {411 return __fetch_sub(*this, __op, __m);412 }413 414 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) volatile noexcept415 requires __base::is_always_lock_free416 {417 return fetch_add(__op) + __op;418 }419 420 _LIBCPP_HIDE_FROM_ABI _Tp operator+=(_Tp __op) noexcept { return fetch_add(__op) + __op; }421 422 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) volatile noexcept423 requires __base::is_always_lock_free424 {425 return fetch_sub(__op) - __op;426 }427 428 _LIBCPP_HIDE_FROM_ABI _Tp operator-=(_Tp __op) noexcept { return fetch_sub(__op) - __op; }429};430 431#endif // _LIBCPP_STD_VER >= 20432 433// atomic_is_lock_free434 435template <class _Tp>436_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const volatile atomic<_Tp>* __o) _NOEXCEPT {437 return __o->is_lock_free();438}439 440template <class _Tp>441_LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const atomic<_Tp>* __o) _NOEXCEPT {442 return __o->is_lock_free();443}444 445// atomic_init446 447template <class _Tp>448_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void449atomic_init(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {450 std::__cxx_atomic_init(std::addressof(__o->__a_), __d);451}452 453template <class _Tp>454_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void455atomic_init(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {456 std::__cxx_atomic_init(std::addressof(__o->__a_), __d);457}458 459// atomic_store460 461template <class _Tp>462_LIBCPP_HIDE_FROM_ABI void atomic_store(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {463 __o->store(__d);464}465 466template <class _Tp>467_LIBCPP_HIDE_FROM_ABI void atomic_store(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {468 __o->store(__d);469}470 471// atomic_store_explicit472 473template <class _Tp>474_LIBCPP_HIDE_FROM_ABI void475atomic_store_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT476 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {477 __o->store(__d, __m);478}479 480template <class _Tp>481_LIBCPP_HIDE_FROM_ABI void482atomic_store_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT483 _LIBCPP_CHECK_STORE_MEMORY_ORDER(__m) {484 __o->store(__d, __m);485}486 487// atomic_load488 489template <class _Tp>490_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const volatile atomic<_Tp>* __o) _NOEXCEPT {491 return __o->load();492}493 494template <class _Tp>495_LIBCPP_HIDE_FROM_ABI _Tp atomic_load(const atomic<_Tp>* __o) _NOEXCEPT {496 return __o->load();497}498 499// atomic_load_explicit500 501template <class _Tp>502_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const volatile atomic<_Tp>* __o, memory_order __m) _NOEXCEPT503 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {504 return __o->load(__m);505}506 507template <class _Tp>508_LIBCPP_HIDE_FROM_ABI _Tp atomic_load_explicit(const atomic<_Tp>* __o, memory_order __m) _NOEXCEPT509 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {510 return __o->load(__m);511}512 513// atomic_exchange514 515template <class _Tp>516_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {517 return __o->exchange(__d);518}519 520template <class _Tp>521_LIBCPP_HIDE_FROM_ABI _Tp atomic_exchange(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d) _NOEXCEPT {522 return __o->exchange(__d);523}524 525// atomic_exchange_explicit526 527template <class _Tp>528_LIBCPP_HIDE_FROM_ABI _Tp529atomic_exchange_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {530 return __o->exchange(__d, __m);531}532 533template <class _Tp>534_LIBCPP_HIDE_FROM_ABI _Tp535atomic_exchange_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __d, memory_order __m) _NOEXCEPT {536 return __o->exchange(__d, __m);537}538 539// atomic_compare_exchange_weak540 541template <class _Tp>542_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(543 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {544 return __o->compare_exchange_weak(*__e, __d);545}546 547template <class _Tp>548_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak(549 atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {550 return __o->compare_exchange_weak(*__e, __d);551}552 553// atomic_compare_exchange_strong554 555template <class _Tp>556_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(557 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {558 return __o->compare_exchange_strong(*__e, __d);559}560 561template <class _Tp>562_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong(563 atomic<_Tp>* __o, typename atomic<_Tp>::value_type* __e, typename atomic<_Tp>::value_type __d) _NOEXCEPT {564 return __o->compare_exchange_strong(*__e, __d);565}566 567// atomic_compare_exchange_weak_explicit568 569template <class _Tp>570_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(571 volatile atomic<_Tp>* __o,572 typename atomic<_Tp>::value_type* __e,573 typename atomic<_Tp>::value_type __d,574 memory_order __s,575 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {576 return __o->compare_exchange_weak(*__e, __d, __s, __f);577}578 579template <class _Tp>580_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(581 atomic<_Tp>* __o,582 typename atomic<_Tp>::value_type* __e,583 typename atomic<_Tp>::value_type __d,584 memory_order __s,585 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {586 return __o->compare_exchange_weak(*__e, __d, __s, __f);587}588 589// atomic_compare_exchange_strong_explicit590 591template <class _Tp>592_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(593 volatile atomic<_Tp>* __o,594 typename atomic<_Tp>::value_type* __e,595 typename atomic<_Tp>::value_type __d,596 memory_order __s,597 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {598 return __o->compare_exchange_strong(*__e, __d, __s, __f);599}600 601template <class _Tp>602_LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(603 atomic<_Tp>* __o,604 typename atomic<_Tp>::value_type* __e,605 typename atomic<_Tp>::value_type __d,606 memory_order __s,607 memory_order __f) _NOEXCEPT _LIBCPP_CHECK_EXCHANGE_MEMORY_ORDER(__s, __f) {608 return __o->compare_exchange_strong(*__e, __d, __s, __f);609}610 611#if _LIBCPP_STD_VER >= 20612 613// atomic_wait614 615template <class _Tp>616_LIBCPP_HIDE_FROM_ABI void617atomic_wait(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {618 return __o->wait(__v);619}620 621template <class _Tp>622_LIBCPP_HIDE_FROM_ABI void atomic_wait(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v) _NOEXCEPT {623 return __o->wait(__v);624}625 626// atomic_wait_explicit627 628template <class _Tp>629_LIBCPP_HIDE_FROM_ABI void630atomic_wait_explicit(const volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT631 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {632 return __o->wait(__v, __m);633}634 635template <class _Tp>636_LIBCPP_HIDE_FROM_ABI void637atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __v, memory_order __m) _NOEXCEPT638 _LIBCPP_CHECK_LOAD_MEMORY_ORDER(__m) {639 return __o->wait(__v, __m);640}641 642// atomic_notify_one643 644template <class _Tp>645_LIBCPP_HIDE_FROM_ABI void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT {646 __o->notify_one();647}648template <class _Tp>649_LIBCPP_HIDE_FROM_ABI void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT {650 __o->notify_one();651}652 653// atomic_notify_all654 655template <class _Tp>656_LIBCPP_HIDE_FROM_ABI void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT {657 __o->notify_all();658}659template <class _Tp>660_LIBCPP_HIDE_FROM_ABI void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT {661 __o->notify_all();662}663 664#endif // _LIBCPP_STD_VER >= 20665 666// atomic_fetch_add667 668template <class _Tp>669_LIBCPP_HIDE_FROM_ABI _Tp670atomic_fetch_add(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {671 return __o->fetch_add(__op);672}673 674template <class _Tp>675_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {676 return __o->fetch_add(__op);677}678 679// atomic_fetch_add_explicit680 681template <class _Tp>682_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_add_explicit(683 volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {684 return __o->fetch_add(__op, __m);685}686 687template <class _Tp>688_LIBCPP_HIDE_FROM_ABI _Tp689atomic_fetch_add_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {690 return __o->fetch_add(__op, __m);691}692 693// atomic_fetch_sub694 695template <class _Tp>696_LIBCPP_HIDE_FROM_ABI _Tp697atomic_fetch_sub(volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {698 return __o->fetch_sub(__op);699}700 701template <class _Tp>702_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op) _NOEXCEPT {703 return __o->fetch_sub(__op);704}705 706// atomic_fetch_sub_explicit707 708template <class _Tp>709_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_sub_explicit(710 volatile atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {711 return __o->fetch_sub(__op, __m);712}713 714template <class _Tp>715_LIBCPP_HIDE_FROM_ABI _Tp716atomic_fetch_sub_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::difference_type __op, memory_order __m) _NOEXCEPT {717 return __o->fetch_sub(__op, __m);718}719 720// atomic_fetch_and721 722template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>723_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {724 return __o->fetch_and(__op);725}726 727template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>728_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {729 return __o->fetch_and(__op);730}731 732// atomic_fetch_and_explicit733 734template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>735_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_and_explicit(736 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {737 return __o->fetch_and(__op, __m);738}739 740template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>741_LIBCPP_HIDE_FROM_ABI _Tp742atomic_fetch_and_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {743 return __o->fetch_and(__op, __m);744}745 746// atomic_fetch_or747 748template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>749_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {750 return __o->fetch_or(__op);751}752 753template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>754_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_or(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {755 return __o->fetch_or(__op);756}757 758// atomic_fetch_or_explicit759 760template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>761_LIBCPP_HIDE_FROM_ABI _Tp762atomic_fetch_or_explicit(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {763 return __o->fetch_or(__op, __m);764}765 766template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>767_LIBCPP_HIDE_FROM_ABI _Tp768atomic_fetch_or_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {769 return __o->fetch_or(__op, __m);770}771 772// atomic_fetch_xor773 774template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>775_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {776 return __o->fetch_xor(__op);777}778 779template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>780_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op) _NOEXCEPT {781 return __o->fetch_xor(__op);782}783 784// atomic_fetch_xor_explicit785 786template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>787_LIBCPP_HIDE_FROM_ABI _Tp atomic_fetch_xor_explicit(788 volatile atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {789 return __o->fetch_xor(__op, __m);790}791 792template <class _Tp, __enable_if_t<is_integral<_Tp>::value && !is_same<_Tp, bool>::value, int> = 0>793_LIBCPP_HIDE_FROM_ABI _Tp794atomic_fetch_xor_explicit(atomic<_Tp>* __o, typename atomic<_Tp>::value_type __op, memory_order __m) _NOEXCEPT {795 return __o->fetch_xor(__op, __m);796}797 798_LIBCPP_END_NAMESPACE_STD799 800#endif // _LIBCPP___ATOMIC_ATOMIC_H801