diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index 9df40eab678a2..914601d2ec651 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -214,6 +214,7 @@ set(files __atomic/atomic_lock_free.h __atomic/atomic_ref.h __atomic/atomic_sync.h + __atomic/atomic_waitable_traits.h __atomic/check_memory_order.h __atomic/contention_t.h __atomic/fence.h diff --git a/libcxx/include/__atomic/atomic.h b/libcxx/include/__atomic/atomic.h index 554c111d695f2..b98620a864052 100644 --- a/libcxx/include/__atomic/atomic.h +++ b/libcxx/include/__atomic/atomic.h @@ -10,6 +10,7 @@ #define _LIBCPP___ATOMIC_ATOMIC_H #include <__atomic/atomic_sync.h> +#include <__atomic/atomic_waitable_traits.h> #include <__atomic/check_memory_order.h> #include <__atomic/floating_point_helper.h> #include <__atomic/is_always_lock_free.h> @@ -200,6 +201,7 @@ struct __atomic_base<_Tp, true> : public __atomic_base<_Tp, false> { _LIBCPP_HIDE_FROM_ABI _Tp operator^=(_Tp __op) _NOEXCEPT { return fetch_xor(__op) ^ __op; } }; +#if _LIBCPP_STD_VER >= 20 // Here we need _IsIntegral because the default template argument is not enough // e.g __atomic_base is __atomic_base, which inherits from // __atomic_base and the caller of the wait function is @@ -228,6 +230,8 @@ struct __atomic_waitable_traits<__atomic_base<_Tp, _IsIntegral> > { } }; +#endif // _LIBCPP_STD_VER >= 20 + template struct __check_atomic_mandates { using type _LIBCPP_NODEBUG = _Tp; @@ -321,10 +325,10 @@ struct atomic<_Tp*> : public __atomic_base<_Tp*> { atomic& operator=(const atomic&) volatile = delete; }; +#if _LIBCPP_STD_VER >= 20 template struct __atomic_waitable_traits > : __atomic_waitable_traits<__atomic_base<_Tp> > {}; -#if _LIBCPP_STD_VER >= 20 template requires is_floating_point_v<_Tp> struct atomic<_Tp> : __atomic_base<_Tp> { diff --git a/libcxx/include/__atomic/atomic_flag.h b/libcxx/include/__atomic/atomic_flag.h index 321a6283ba7ad..42864c869d22f 100644 --- a/libcxx/include/__atomic/atomic_flag.h +++ b/libcxx/include/__atomic/atomic_flag.h @@ -10,6 +10,7 @@ #define _LIBCPP___ATOMIC_ATOMIC_FLAG_H #include <__atomic/atomic_sync.h> +#include <__atomic/atomic_waitable_traits.h> #include <__atomic/contention_t.h> #include <__atomic/memory_order.h> #include <__atomic/support.h> @@ -74,6 +75,7 @@ struct atomic_flag { atomic_flag& operator=(const atomic_flag&) volatile = delete; }; +#if _LIBCPP_STD_VER >= 20 template <> struct __atomic_waitable_traits { using __value_type _LIBCPP_NODEBUG = _LIBCPP_ATOMIC_FLAG_TYPE; @@ -97,6 +99,7 @@ struct __atomic_waitable_traits { return std::addressof(__a.__a_); } }; +#endif // _LIBCPP_STD_VER >= 20 inline _LIBCPP_HIDE_FROM_ABI bool atomic_flag_test(const volatile atomic_flag* __o) _NOEXCEPT { return __o->test(); } diff --git a/libcxx/include/__atomic/atomic_ref.h b/libcxx/include/__atomic/atomic_ref.h index 9a36aaa3b84fe..b00685f7ce74c 100644 --- a/libcxx/include/__atomic/atomic_ref.h +++ b/libcxx/include/__atomic/atomic_ref.h @@ -19,6 +19,7 @@ #include <__assert> #include <__atomic/atomic_sync.h> +#include <__atomic/atomic_waitable_traits.h> #include <__atomic/check_memory_order.h> #include <__atomic/floating_point_helper.h> #include <__atomic/memory_order.h> diff --git a/libcxx/include/__atomic/atomic_sync.h b/libcxx/include/__atomic/atomic_sync.h index d0119ab5d35ec..9f46dfe65bb0d 100644 --- a/libcxx/include/__atomic/atomic_sync.h +++ b/libcxx/include/__atomic/atomic_sync.h @@ -9,6 +9,7 @@ #ifndef _LIBCPP___ATOMIC_ATOMIC_SYNC_H #define _LIBCPP___ATOMIC_ATOMIC_SYNC_H +#include <__atomic/atomic_waitable_traits.h> #include <__atomic/contention_t.h> #include <__atomic/memory_order.h> #include <__atomic/to_gcc_order.h> @@ -18,10 +19,8 @@ #include <__thread/poll_with_backoff.h> #include <__type_traits/conjunction.h> #include <__type_traits/decay.h> -#include <__type_traits/has_unique_object_representation.h> #include <__type_traits/invoke.h> #include <__type_traits/is_same.h> -#include <__type_traits/is_trivially_copyable.h> #include <__type_traits/void_t.h> #include <__utility/declval.h> #include @@ -32,34 +31,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD -// The customisation points to enable the following functions: -// - __atomic_wait -// - __atomic_wait_unless -// - __atomic_notify_one -// - __atomic_notify_all -// Note that std::atomic::wait was back-ported to C++03 -// The below implementations look ugly to support C++03 -template -struct __atomic_waitable_traits { - using __value_type _LIBCPP_NODEBUG = void; - - template - static void __atomic_load(_AtomicWaitable&&, memory_order) = delete; - - template - static void __atomic_contention_address(_AtomicWaitable&&) = delete; -}; - -template -struct __atomic_waitable : false_type {}; - -template -struct __atomic_waitable< _Tp, - __void_t >::__atomic_load( - std::declval(), std::declval())), - decltype(__atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address( - std::declval()))> > : true_type {}; - #if _LIBCPP_STD_VER >= 20 # if _LIBCPP_HAS_THREADS @@ -108,48 +79,6 @@ _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_one template _LIBCPP_AVAILABILITY_NEW_SYNC _LIBCPP_EXPORTED_FROM_ABI void __atomic_notify_all_native(const void*) _NOEXCEPT; -# ifdef __linux__ -# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(4) -# elif defined(__APPLE__) -# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) \ - _APPLY(4) \ - _APPLY(8) -# elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8 -# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8) -# elif defined(_WIN32) -# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8) -# else -# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(sizeof(__cxx_contention_t)) -# endif // __linux__ - -// concepts defines the types are supported natively by the platform's wait - -# if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE) - -_LIBCPP_HIDE_FROM_ABI constexpr bool __has_native_atomic_wait_impl(size_t __size) { - switch (__size) { -# define _LIBCPP_MAKE_CASE(n) \ - case n: \ - return true; - _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_LIBCPP_MAKE_CASE) - default: - return false; -# undef _LIBCPP_MAKE_CASE - }; -} - -template -concept __has_native_atomic_wait = - has_unique_object_representations_v<_Tp> && is_trivially_copyable_v<_Tp> && - __has_native_atomic_wait_impl(sizeof(_Tp)); - -# else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE - -template -concept __has_native_atomic_wait = is_same_v<_Tp, __cxx_contention_t>; - -# endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE - # if _LIBCPP_AVAILABILITY_HAS_NEW_SYNC template @@ -240,7 +169,7 @@ struct __atomic_wait_backoff_impl { // value. The predicate function must not return `false` spuriously. template _LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, memory_order __order, _Poll&& __poll) { - static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + static_assert(__atomic_waitable<_AtomicWaitable>); __atomic_wait_backoff_impl<_AtomicWaitable, __decay_t<_Poll> > __backoff_fn = {__a, __poll, __order}; std::__libcpp_thread_poll_with_backoff( /* poll */ @@ -255,7 +184,7 @@ _LIBCPP_HIDE_FROM_ABI void __atomic_wait_unless(const _AtomicWaitable& __a, memo template _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) { - static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + static_assert(__atomic_waitable<_AtomicWaitable>); using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type; using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >; auto __contention_address = @@ -269,7 +198,7 @@ _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) { template _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) { - static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + static_assert(__atomic_waitable<_AtomicWaitable>); using __value_type _LIBCPP_NODEBUG = typename __atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__value_type; using __waitable_traits _LIBCPP_NODEBUG = __atomic_waitable_traits<__decay_t<_AtomicWaitable> >; auto __contention_address = @@ -285,13 +214,13 @@ _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) { template _LIBCPP_HIDE_FROM_ABI void __atomic_notify_one(const _AtomicWaitable& __a) { - static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + static_assert(__atomic_waitable<_AtomicWaitable>); std::__cxx_atomic_notify_one(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a)); } template _LIBCPP_HIDE_FROM_ABI void __atomic_notify_all(const _AtomicWaitable& __a) { - static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + static_assert(__atomic_waitable<_AtomicWaitable>); std::__cxx_atomic_notify_all(__atomic_waitable_traits<__decay_t<_AtomicWaitable> >::__atomic_contention_address(__a)); } @@ -325,7 +254,7 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_nonatomic_compare_equal(_Tp const& __lhs, _Tp c template _LIBCPP_HIDE_FROM_ABI void __atomic_wait(_AtomicWaitable& __a, _Tp __val, memory_order __order) { - static_assert(__atomic_waitable<_AtomicWaitable>::value, ""); + static_assert(__atomic_waitable<_AtomicWaitable>); std::__atomic_wait_unless(__a, __order, [&](_Tp const& __current) { return !std::__cxx_nonatomic_compare_equal(__current, __val); }); diff --git a/libcxx/include/__atomic/atomic_waitable_traits.h b/libcxx/include/__atomic/atomic_waitable_traits.h new file mode 100644 index 0000000000000..de06fe70b3e1a --- /dev/null +++ b/libcxx/include/__atomic/atomic_waitable_traits.h @@ -0,0 +1,100 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H +#define _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H + +#include <__atomic/contention_t.h> +#include <__atomic/memory_order.h> +#include <__config> +#include <__type_traits/decay.h> +#include <__type_traits/has_unique_object_representation.h> +#include <__type_traits/is_same.h> +#include <__type_traits/is_trivially_copyable.h> +#include <__type_traits/void_t.h> +#include <__utility/declval.h> +#include + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +#if _LIBCPP_STD_VER >= 20 + +// The customisation points to enable the following functions: +// - __atomic_wait +// - __atomic_wait_unless +// - __atomic_notify_one +// - __atomic_notify_all +template +struct __atomic_waitable_traits { + using __value_type _LIBCPP_NODEBUG = void; + + template + static void __atomic_load(_AtomicWaitable&&, memory_order) = delete; + + template + static void __atomic_contention_address(_AtomicWaitable&&) = delete; +}; + +template +concept __atomic_waitable = requires(const _Tp __t, memory_order __order) { + typename __atomic_waitable_traits<__decay_t<_Tp> >::__value_type; + { __atomic_waitable_traits<__decay_t<_Tp> >::__atomic_load(__t, __order) }; + { __atomic_waitable_traits<__decay_t<_Tp> >::__atomic_contention_address(__t) }; +}; + +# ifdef __linux__ +# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(4) +# elif defined(__APPLE__) +# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) \ + _APPLY(4) \ + _APPLY(8) +# elif defined(__FreeBSD__) && __SIZEOF_LONG__ == 8 +# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8) +# elif defined(_WIN32) +# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(8) +# else +# define _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_APPLY) _APPLY(sizeof(__cxx_contention_t)) +# endif // __linux__ + +// concepts defines the types are supported natively by the platform's wait + +# if defined(_LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE) + +_LIBCPP_HIDE_FROM_ABI constexpr bool __has_native_atomic_wait_impl(size_t __size) { + switch (__size) { +# define _LIBCPP_MAKE_CASE(n) \ + case n: \ + return true; + _LIBCPP_NATIVE_PLATFORM_WAIT_SIZES(_LIBCPP_MAKE_CASE) + default: + return false; +# undef _LIBCPP_MAKE_CASE + }; +} + +template +concept __has_native_atomic_wait = + has_unique_object_representations_v<_Tp> && is_trivially_copyable_v<_Tp> && + __has_native_atomic_wait_impl(sizeof(_Tp)); + +# else // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE + +template +concept __has_native_atomic_wait = is_same_v<_Tp, __cxx_contention_t>; + +# endif // _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE + +#endif // C++20 + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___ATOMIC_ATOMIC_WAITABLE_TRAITS_H diff --git a/libcxx/include/atomic b/libcxx/include/atomic index 75af5de33ca4c..3323650f7f81a 100644 --- a/libcxx/include/atomic +++ b/libcxx/include/atomic @@ -608,6 +608,7 @@ template # include <__atomic/atomic_init.h> # include <__atomic/atomic_lock_free.h> # include <__atomic/atomic_sync.h> +# include <__atomic/atomic_waitable_traits.h> # include <__atomic/check_memory_order.h> # include <__atomic/contention_t.h> # include <__atomic/fence.h> diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in index ce168f77dfea4..e24e83420405d 100644 --- a/libcxx/include/module.modulemap.in +++ b/libcxx/include/module.modulemap.in @@ -869,6 +869,7 @@ module std [system] { module atomic_lock_free { header "__atomic/atomic_lock_free.h" } module atomic_ref { header "__atomic/atomic_ref.h" } module atomic_sync { header "__atomic/atomic_sync.h" } + module atomic_waitable_traits { header "__atomic/atomic_waitable_traits.h" } module atomic { header "__atomic/atomic.h" export std.atomic.atomic_base // most of std::atomic methods are defined there