[libc++] Implement P0528R3 `std::atomic` CAS for types with padding by huixie90 · Pull Request #76180 · llvm/llvm-project

huixie90 · 2023-12-21T20:31:49Z

Implement P0528R3 The Curious Case of Padding Bits, Featuring Atomic Compare-and-Exchange

The unit test padding.pass.cpp is assisted by Cursor. However, all tests have been modified to test what I wanted to test

llvmbot · 2023-12-21T20:32:17Z

@llvm/pr-subscribers-libcxx

Author: Hui (huixie90)

Changes

Patch is 21.72 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/76180.diff

2 Files Affected:

(modified) libcxx/include/__atomic/cxx_atomic_impl.h (+190-92)
(added) libcxx/test/std/atomics/atomics.types.generic/padding.pass.cpp (+54)

diff --git a/libcxx/include/__atomic/cxx_atomic_impl.h b/libcxx/include/__atomic/cxx_atomic_impl.h
index 1a0b808a0cb1c4..3efdc077a7428a 100644
--- a/libcxx/include/__atomic/cxx_atomic_impl.h
+++ b/libcxx/include/__atomic/cxx_atomic_impl.h
@@ -15,6 +15,7 @@
 #include <__memory/addressof.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/is_assignable.h>
+#include <__type_traits/is_same.h>
 #include <__type_traits/is_trivially_copyable.h>
 #include <__type_traits/remove_const.h>
 #include <cstddef>
@@ -26,6 +27,32 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+template <class _Tp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR void __clear_padding_if_needed(_Tp* __ptr) noexcept {
+#if _LIBCPP_STD_VER >= 20
+  constexpr bool __needs_clear_padding =
+      !__has_unique_object_representations(_Tp) && !is_same<_Tp, float>::value && !is_same<_Tp, double>::value;
+  if constexpr (__needs_clear_padding) {
+    if (!__builtin_is_constant_evaluated()) {
+      __builtin_clear_padding(__ptr);
+    }
+  }
+#endif
+}
+
+template <typename _Tp, typename _ImplFunc>
+_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_impl(_Tp* __expected, _Tp __value, _ImplFunc&& __impl_fun) {
+  std::__clear_padding_if_needed(std::addressof(__value));
+  _Tp __expected_copy = *__expected;
+  std::__clear_padding_if_needed(std::addressof(__expected_copy));
+  if (__impl_fun(std::addressof(__expected_copy), __value)) {
+    return true;
+  } else {
+    std::memcpy(std::addressof(__expected_copy), __expected, sizeof(_Tp));
+    return false;
+  }
+}
+
 #if defined(_LIBCPP_HAS_GCC_ATOMIC_IMP) || defined(_LIBCPP_ATOMIC_ONLY_USE_BUILTINS)
 
 // [atomics.types.generic]p1 guarantees _Tp is trivially copyable. Because
@@ -52,12 +79,20 @@ template <typename _Tp>
 struct __cxx_atomic_base_impl {
   _LIBCPP_HIDE_FROM_ABI
 #  ifndef _LIBCPP_CXX03_LANG
+#    if _LIBCPP_STD_VER >= 20
+  __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
+    std::__clear_padding_if_needed(std::addressof(__a_value));
+  }
+#    else
   __cxx_atomic_base_impl() _NOEXCEPT = default;
+#    endif
 #  else
   __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
   }
 #  endif // _LIBCPP_CXX03_LANG
-  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {}
+  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {
+    std::__clear_padding_if_needed(std::addressof(__a_value));
+  }
   _Tp __a_value;
 };
 
@@ -108,11 +143,13 @@ _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI void
 __cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
+  std::__clear_padding_if_needed(std::addressof(__val));
   __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
 }
 
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
+  std::__clear_padding_if_needed(std::addressof(__val));
   __atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
 }
 
@@ -146,6 +183,7 @@ template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI _Tp
 __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
   _Tp __ret;
+  std::__clear_padding_if_needed(std::addressof(__value));
   __atomic_exchange(
       std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
   return __ret;
@@ -154,6 +192,7 @@ __cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, me
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
   _Tp __ret;
+  std::__clear_padding_if_needed(std::addressof(__value));
   __atomic_exchange(
       std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
   return __ret;
@@ -166,25 +205,31 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
     _Tp __value,
     memory_order __success,
     memory_order __failure) {
-  return __atomic_compare_exchange(
-      std::addressof(__a->__a_value),
-      __expected,
-      std::addressof(__value),
-      false,
-      __to_gcc_order(__success),
-      __to_gcc_failure_order(__failure));
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        return __atomic_compare_exchange(
+            std::addressof(__a->__a_value),
+            __expected_cleared_padding,
+            __value_cleared_padding,
+            false,
+            __to_gcc_order(__success),
+            __to_gcc_failure_order(__failure));
+      });
 }
 
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
     __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
-  return __atomic_compare_exchange(
-      std::addressof(__a->__a_value),
-      __expected,
-      std::addressof(__value),
-      false,
-      __to_gcc_order(__success),
-      __to_gcc_failure_order(__failure));
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        return __atomic_compare_exchange(
+            std::addressof(__a->__a_value),
+            __expected_cleared_padding,
+            __value_cleared_padding,
+            false,
+            __to_gcc_order(__success),
+            __to_gcc_failure_order(__failure));
+      });
 }
 
 template <typename _Tp>
@@ -194,25 +239,31 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
     _Tp __value,
     memory_order __success,
     memory_order __failure) {
-  return __atomic_compare_exchange(
-      std::addressof(__a->__a_value),
-      __expected,
-      std::addressof(__value),
-      true,
-      __to_gcc_order(__success),
-      __to_gcc_failure_order(__failure));
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        return __atomic_compare_exchange(
+            std::addressof(__a->__a_value),
+            __expected_cleared_padding,
+            __value_cleared_padding,
+            true,
+            __to_gcc_order(__success),
+            __to_gcc_failure_order(__failure));
+      });
 }
 
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
     __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
-  return __atomic_compare_exchange(
-      std::addressof(__a->__a_value),
-      __expected,
-      std::addressof(__value),
-      true,
-      __to_gcc_order(__success),
-      __to_gcc_failure_order(__failure));
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        return __atomic_compare_exchange(
+            std::addressof(__a->__a_value),
+            __expected_cleared_padding,
+            __value_cleared_padding,
+            true,
+            __to_gcc_order(__success),
+            __to_gcc_failure_order(__failure));
+      });
 }
 
 template <typename _Tp>
@@ -297,12 +348,20 @@ template <typename _Tp>
 struct __cxx_atomic_base_impl {
   _LIBCPP_HIDE_FROM_ABI
 #  ifndef _LIBCPP_CXX03_LANG
+#    if _LIBCPP_STD_VER >= 20
+  __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
+    std::__clear_padding_if_needed(std::addressof(__a_value));
+  }
+#    else
   __cxx_atomic_base_impl() _NOEXCEPT = default;
+#    endif
 #  else
   __cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
   }
 #  endif // _LIBCPP_CXX03_LANG
-  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
+  _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {
+    std::__clear_padding_if_needed(std::addressof(__a_value));
+  }
   _LIBCPP_DISABLE_EXTENSION_WARNING _Atomic(_Tp) __a_value;
 };
 
@@ -328,11 +387,13 @@ _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI void
 __cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
+  std::__clear_padding_if_needed(std::addressof(__val));
   __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
 }
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI void
 __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
+  std::__clear_padding_if_needed(std::addressof(__val));
   __c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
 }
 
@@ -368,12 +429,14 @@ __cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, me
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI _Tp
 __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
+  std::__clear_padding_if_needed(std::addressof(__value));
   return __c11_atomic_exchange(
       std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
 }
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI _Tp
 __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
+  std::__clear_padding_if_needed(std::addressof(__value));
   return __c11_atomic_exchange(
       std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
 }
@@ -392,23 +455,30 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
     _Tp __value,
     memory_order __success,
     memory_order __failure) _NOEXCEPT {
-  return __c11_atomic_compare_exchange_strong(
-      std::addressof(__a->__a_value),
-      __expected,
-      __value,
-      static_cast<__memory_order_underlying_t>(__success),
-      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        return __c11_atomic_compare_exchange_strong(
+            std::addressof(__a->__a_value),
+            __expected_cleared_padding,
+            __value_cleared_padding,
+            static_cast<__memory_order_underlying_t>(__success),
+            static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+      });
 }
+
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
     __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
     _NOEXCEPT {
-  return __c11_atomic_compare_exchange_strong(
-      std::addressof(__a->__a_value),
-      __expected,
-      __value,
-      static_cast<__memory_order_underlying_t>(__success),
-      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        return __c11_atomic_compare_exchange_strong(
+            std::addressof(__a->__a_value),
+            __expected_cleared_padding,
+            __value_cleared_padding,
+            static_cast<__memory_order_underlying_t>(__success),
+            static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+      });
 }
 
 template <class _Tp>
@@ -418,23 +488,29 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
     _Tp __value,
     memory_order __success,
     memory_order __failure) _NOEXCEPT {
-  return __c11_atomic_compare_exchange_weak(
-      std::addressof(__a->__a_value),
-      __expected,
-      __value,
-      static_cast<__memory_order_underlying_t>(__success),
-      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        return __c11_atomic_compare_exchange_weak(
+            std::addressof(__a->__a_value),
+            __expected_cleared_padding,
+            __value_cleared_padding,
+            static_cast<__memory_order_underlying_t>(__success),
+            static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+      });
 }
 template <class _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
     __cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
     _NOEXCEPT {
-  return __c11_atomic_compare_exchange_weak(
-      std::addressof(__a->__a_value),
-      __expected,
-      __value,
-      static_cast<__memory_order_underlying_t>(__success),
-      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        return __c11_atomic_compare_exchange_weak(
+            std::addressof(__a->__a_value),
+            __expected_cleared_padding,
+            __value_cleared_padding,
+            static_cast<__memory_order_underlying_t>(__success),
+            static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+      });
 }
 
 template <class _Tp>
@@ -533,10 +609,14 @@ __cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_o
 
 template <typename _Tp>
 struct __cxx_atomic_lock_impl {
-  _LIBCPP_HIDE_FROM_ABI __cxx_atomic_lock_impl() _NOEXCEPT : __a_value(), __a_lock(0) {}
+  _LIBCPP_HIDE_FROM_ABI __cxx_atomic_lock_impl() _NOEXCEPT : __a_value(), __a_lock(0) {
+    std::__clear_padding_if_needed(std::addressof(__a_value));
+  }
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_lock_impl(_Tp value) _NOEXCEPT
       : __a_value(value),
-        __a_lock(0) {}
+        __a_lock(0) {
+    std::__clear_padding_if_needed(std::addressof(__a_value));
+  }
 
   _Tp __a_value;
   mutable __cxx_atomic_base_impl<_LIBCPP_ATOMIC_FLAG_TYPE> __a_lock;
@@ -592,12 +672,14 @@ _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_lock_impl<_Tp>* __a, _
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) {
   __a->__lock();
+  std::__clear_padding_if_needed(std::addressof(__val));
   __cxx_atomic_assign_volatile(__a->__a_value, __val);
   __a->__unlock();
 }
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __val, memory_order) {
   __a->__lock();
+  std::__clear_padding_if_needed(std::addressof(__val));
   __a->__a_value = __val;
   __a->__unlock();
 }
@@ -625,6 +707,7 @@ template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
   __a->__lock();
   _Tp __old;
+  std::__clear_padding_if_needed(std::addressof(__value));
   __cxx_atomic_assign_volatile(__old, __a->__a_value);
   __cxx_atomic_assign_volatile(__a->__a_value, __value);
   __a->__unlock();
@@ -633,6 +716,7 @@ _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(volatile __cxx_atomic_lock_impl<
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a, _Tp __value, memory_order) {
   __a->__lock();
+  std::__clear_padding_if_needed(std::addressof(__value));
   _Tp __old      = __a->__a_value;
   __a->__a_value = __value;
   __a->__unlock();
@@ -642,55 +726,69 @@ _LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_lock_impl<_Tp>* __a
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
     volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) {
-  _Tp __temp;
-  __a->__lock();
-  __cxx_atomic_assign_volatile(__temp, __a->__a_value);
-  bool __ret = (std::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
-  if (__ret)
-    __cxx_atomic_assign_volatile(__a->__a_value, __value);
-  else
-    __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
-  __a->__unlock();
-  return __ret;
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        _Tp __temp;
+        __a->__lock();
+        __cxx_atomic_assign_volatile(__temp, __a->__a_value);
+        bool __ret = (std::memcmp(&__temp, __expected_cleared_padding, sizeof(_Tp)) == 0);
+        if (__ret)
+          __cxx_atomic_assign_volatile(__a->__a_value, __value_cleared_padding);
+        else
+          __cxx_atomic_assign_volatile(*__expected_cleared_padding, __a->__a_value);
+        __a->__unlock();
+        return __ret;
+      });
 }
+
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
     __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) {
-  __a->__lock();
-  bool __ret = (std::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
-  if (__ret)
-    std::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
-  else
-    std::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
-  __a->__unlock();
-  return __ret;
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        __a->__lock();
+        bool __ret = (std::memcmp(&__a->__a_value, __expected_cleared_padding, sizeof(_Tp)) == 0);
+        if (__ret)
+          std::memcpy(&__a->__a_value, &__value_cleared_padding, sizeof(_Tp));
+        else
+          std::memcpy(__expected_cleared_padding, &__a->__a_value, sizeof(_Tp));
+        __a->__unlock();
+        return __ret;
+      });
 }
 
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
     volatile __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) {
-  _Tp __temp;
-  __a->__lock();
-  __cxx_atomic_assign_volatile(__temp, __a->__a_value);
-  bool __ret = (std::memcmp(&__temp, __expected, sizeof(_Tp)) == 0);
-  if (__ret)
-    __cxx_atomic_assign_volatile(__a->__a_value, __value);
-  else
-    __cxx_atomic_assign_volatile(*__expected, __a->__a_value);
-  __a->__unlock();
-  return __ret;
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        _Tp __temp;
+        __a->__lock();
+        __cxx_atomic_assign_volatile(__temp, __a->__a_value);
+        bool __ret = (std::memcmp(&__temp, __expected_cleared_padding, sizeof(_Tp)) == 0);
+        if (__ret)
+          __cxx_atomic_assign_volatile(__a->__a_value, __value_cleared_padding);
+        else
+          __cxx_atomic_assign_volatile(*__expected_cleared_padding, __a->__a_value);
+        __a->__unlock();
+        return __ret;
+      });
 }
+
 template <typename _Tp>
 _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
     __cxx_atomic_lock_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order, memory_order) {
-  __a->__lock();
-  bool __ret = (std::memcmp(&__a->__a_value, __expected, sizeof(_Tp)) == 0);
-  if (__ret)
-    std::memcpy(&__a->__a_value, &__value, sizeof(_Tp));
-  else
-    std::memcpy(__expected, &__a->__a_value, sizeof(_Tp));
-  __a->__unlock();
-  return __ret;
+  return std::__cxx_atomic_compare_exchange_impl(
+      __expected, __value, [__a, __success, __failure](_Tp* __expected_cleared_padding, _Tp& __value_cleared_padding) {
+        __a->__lock();
+        bool __ret = (std::memcmp(&__a->__a_value, __expected_cleared_padding, sizeof(_Tp)) == 0);
+        if (__ret)
+          std::memcpy(&__a->__a_value, &__value_cleared_padding, sizeof(_Tp));
+        else
+          std::memcpy(__expected_cleared_padding,...
[truncated]

mordante

I mainly glossed over the patch since it's still a draft.

mordante · 2023-12-22T19:21:43Z

+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR void __clear_padding_if_needed(_Tp* __ptr) noexcept {
+#if _LIBCPP_STD_VER >= 20
+  constexpr bool __needs_clear_padding =
+      !__has_unique_object_representations(_Tp) && !is_same<_Tp, float>::value && !is_same<_Tp, double>::value;


Is it guaranteed float and double never have padding. Wouldn't it be better to have a builtin that tells whether padding should be removed? Or even make the builtin a no-op when it's not needed for the type.

mordante · 2023-12-22T19:22:00Z

+  constexpr bool __needs_clear_padding =
+      !__has_unique_object_representations(_Tp) && !is_same<_Tp, float>::value && !is_same<_Tp, double>::value;
+  if constexpr (__needs_clear_padding) {
+    if (!__builtin_is_constant_evaluated()) {


why not std::is_constant_evaluated?

) Add builtin to clear padding bytes. This is the pre-work to implement `std::atomic::compare_exchange_[weak/strong]` that ignores padding bits. PR draft here: #76180 This PR picked up this patch from 3 years ago https://reviews.llvm.org/D87974 The above patch no longer works as things changed quite a lot. I've made some changes on top of the above patch: it handles: - struct - builtin types with paddings (like `long double` and types with `__attribute__((ext_vector_type(N)))` - _Complex long double - constant array - union - bit field - types with virtual functions - types with virtual bases --------- Co-authored-by: zoecarver <z.zoelec2@gmail.com>

…528R3) (#75371) Add builtin to clear padding bytes. This is the pre-work to implement `std::atomic::compare_exchange_[weak/strong]` that ignores padding bits. PR draft here: llvm/llvm-project#76180 This PR picked up this patch from 3 years ago https://reviews.llvm.org/D87974 The above patch no longer works as things changed quite a lot. I've made some changes on top of the above patch: it handles: - struct - builtin types with paddings (like `long double` and types with `__attribute__((ext_vector_type(N)))` - _Complex long double - constant array - union - bit field - types with virtual functions - types with virtual bases --------- Co-authored-by: zoecarver <z.zoelec2@gmail.com>

github-actions · 2026-06-07T20:42:22Z

✅ With the latest revision this PR passed the C/C++ code formatter.

ldionne · 2026-06-19T18:20:34Z

CC @jiixyj in case you're interested to help review this

ldionne

I'm glad to see this making progress! This is a non-trivial patch, I have some comments but it's looking very reasonable.

ldionne

LGTM assuming the C++11-ifying works with GCC.

This reverts commit 8be9a83.

Co-authored-by: Louis Dionne <ldionne.2@gmail.com>

ldionne · 2026-07-27T18:21:28Z

+    // for non trivially copyable types, __builtin_clear_padding matches gcc's behaviour
+    // where the pointer to the object must be known to the compiler
+    // using libc++'s __clear_padding_if_needed would make the object "unknown" to the compiler
+    // note that in atomic<T>, T must be trivially copyable
    __builtin_clear_padding(&s2);


Why is this not __clear_padding_if_needed?

Nevermind, I didn't read the comment above carefully enough.

ldionne

LGTM

ldionne · 2026-07-27T18:22:48Z

+    // for non trivially copyable types, __builtin_clear_padding matches gcc's behaviour
+    // where the pointer to the object must be known to the compiler
+    // using libc++'s __clear_padding_if_needed would make the object "unknown" to the compiler
+    // note that in atomic<T>, T must be trivially copyable
    __builtin_clear_padding(&s2);


Nevermind, I didn't read the comment above carefully enough.

philnik777 · 2026-07-28T07:33:44Z

-      static_cast<__memory_order_underlying_t>(__success),
-      static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
+      // Use always_inline since Clang fails to inline the lambda, even when the memory order is a constant
+      [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {


Why is this marked with _LIBCPP_ALWAYS_INLINE?

Edit: Sorry, missed the comment above.

Ah Ok. Let me know if the comment is unclear. The detailed discussion is here #76180 (comment)

…lvm#76180) Fixes llvm#99984 Implement P0528R3 The Curious Case of Padding Bits, Featuring Atomic Compare-and-Exchange The unit test padding.pass.cpp is assisted by Cursor. However, all tests have been modified to test what I wanted to test --------- Co-authored-by: Louis Dionne <ldionne.2@gmail.com>

huixie90 requested a review from a team as a code owner December 21, 2023 20:31

llvmbot added the libc++ libc++ C++ Standard Library. Not GNU libstdc++. Not libc++abi. label Dec 21, 2023

huixie90 marked this pull request as draft December 21, 2023 20:32

huixie90 mentioned this pull request Dec 21, 2023

[clang] Add builtin to clear padding bytes (prework for P0528R3) #75371

Merged

mordante reviewed Dec 22, 2023

View reviewed changes

huixie90 mentioned this pull request Jan 3, 2024

[libc++] Implement C++20 atomic_ref #76647

Merged

ldionne self-assigned this Jan 10, 2024

ldionne reviewed Jan 19, 2024

View reviewed changes

Comment thread libcxx/include/__atomic/cxx_atomic_impl.h Outdated

huixie90 changed the title ~~[libc++] make std::atomic works with types with paddings~~ [libc++] Implement P0528R3 std::atomic CAS for types with paddings Jun 7, 2026

huixie90 force-pushed the atomic_padding branch from 2df576d to 483a78c Compare June 7, 2026 20:40

huixie90 marked this pull request as ready for review June 7, 2026 20:41

huixie90 force-pushed the atomic_padding branch 4 times, most recently from e5e734b to 1f576b7 Compare June 14, 2026 14:06

huixie90 force-pushed the atomic_padding branch from 1f576b7 to ac972d4 Compare June 15, 2026 11:15

ldionne reviewed Jun 18, 2026

View reviewed changes

Comment thread libcxx/test/std/atomics/atomics.types.generic/padding.pass.cpp Outdated

ldionne requested a review from dalg24 June 19, 2026 18:19

huixie90 force-pushed the atomic_padding branch 2 times, most recently from b208bae to 33732fe Compare June 21, 2026 13:24

ldionne assigned huixie90 and unassigned ldionne Jul 3, 2026

ldionne requested changes Jul 3, 2026

View reviewed changes

ldionne approved these changes Jul 10, 2026

View reviewed changes

huixie90 and others added 16 commits July 27, 2026 17:47

debug

f7b6225

Revert "debug"

8f5626f

This reverts commit 8be9a83.

test ci

40c3316

padding

75b19b2

clang22

399acfd

cxx 03

bc33e49

ci

7b17a5b

ci

5ddfbe6

ci

749277c

ci

5739b6c

ci

2adfb7c

review

e278047

inline

7d841c5

f

3b63e5c

Update libcxx/include/__atomic/support/c11.h

edc991c

Co-authored-by: Louis Dionne <ldionne.2@gmail.com>

Update libcxx/utils/libcxx/test/features/misc.py

6417035

Co-authored-by: Louis Dionne <ldionne.2@gmail.com>

huixie90 force-pushed the atomic_padding branch from 47bd811 to 6417035 Compare July 27, 2026 17:08

review

6a15f6a

ldionne reviewed Jul 27, 2026

View reviewed changes

ldionne approved these changes Jul 27, 2026

View reviewed changes

ldionne changed the title ~~[libc++] Implement P0528R3 std::atomic CAS for types with paddings~~ [libc++] Implement P0528R3 std::atomic CAS for types with padding Jul 27, 2026

huixie90 added 2 commits July 27, 2026 20:17

ci

a546c54

rename

5766ff8

ldionne approved these changes Jul 27, 2026

View reviewed changes

ldionne added the pending-ci Merging the PR is only pending completion of CI label Jul 27, 2026

huixie90 merged commit de962dd into llvm:main Jul 28, 2026
85 checks passed

philnik777 reviewed Jul 28, 2026

View reviewed changes

frederick-vs-ja removed the pending-ci Merging the PR is only pending completion of CI label Jul 28, 2026

Uh oh!

Conversation

huixie90 commented Dec 21, 2023 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

llvmbot commented Dec 21, 2023

Uh oh!

mordante left a comment

Choose a reason for hiding this comment

Uh oh!

mordante Dec 22, 2023

Choose a reason for hiding this comment

Uh oh!

mordante Dec 22, 2023

Choose a reason for hiding this comment

Uh oh!

Uh oh!

github-actions Bot commented Jun 7, 2026 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

Uh oh!

ldionne commented Jun 19, 2026

Uh oh!

ldionne left a comment

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

ldionne left a comment

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

Uh oh!

ldionne Jul 27, 2026

Choose a reason for hiding this comment

Uh oh!

ldionne Jul 27, 2026

Choose a reason for hiding this comment

Uh oh!

Uh oh!

ldionne left a comment

Choose a reason for hiding this comment

Uh oh!

ldionne Jul 27, 2026

Choose a reason for hiding this comment

Uh oh!

Uh oh!

philnik777 Jul 28, 2026 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

huixie90 Jul 28, 2026

Choose a reason for hiding this comment

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

6 participants

huixie90 commented Dec 21, 2023 •

edited

Loading

github-actions Bot commented Jun 7, 2026 •

edited

Loading

philnik777 Jul 28, 2026 •

edited

Loading