diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt index f2f1ec50d5b75..e16bc87e79885 100644 --- a/libcxx/include/CMakeLists.txt +++ b/libcxx/include/CMakeLists.txt @@ -571,6 +571,7 @@ set(files __memory/allocator.h __memory/allocator_arg_t.h __memory/allocator_destructor.h + __memory/allocator_relocation.h __memory/allocator_traits.h __memory/array_cookie.h __memory/assume_aligned.h @@ -589,12 +590,14 @@ set(files __memory/ranges_destroy.h __memory/ranges_uninitialized_algorithms.h __memory/raw_storage_iterator.h + __memory/relocate_at.h __memory/shared_count.h __memory/shared_ptr.h __memory/swap_allocator.h __memory/temp_value.h __memory/temporary_buffer.h __memory/uninitialized_algorithms.h + __memory/uninitialized_relocate.h __memory/unique_ptr.h __memory/unique_temporary_buffer.h __memory/uses_allocator.h @@ -846,6 +849,7 @@ set(files __type_traits/is_nothrow_assignable.h __type_traits/is_nothrow_constructible.h __type_traits/is_nothrow_destructible.h + __type_traits/is_nothrow_relocatable.h __type_traits/is_null_pointer.h __type_traits/is_object.h __type_traits/is_pod.h diff --git a/libcxx/include/__memory/allocator_relocation.h b/libcxx/include/__memory/allocator_relocation.h new file mode 100644 index 0000000000000..62dfa817bf2e5 --- /dev/null +++ b/libcxx/include/__memory/allocator_relocation.h @@ -0,0 +1,196 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___MEMORY_ALLOCATOR_RELOCATION_H +#define _LIBCPP___MEMORY_ALLOCATOR_RELOCATION_H + +#include <__assert> +#include <__config> +#include <__iterator/iterator_traits.h> +#include <__memory/addressof.h> +#include <__memory/allocator_traits.h> +#include <__memory/destroy.h> +#include <__memory/pointer_traits.h> +#include <__memory/relocate_at.h> +#include <__memory/uninitialized_relocate.h> +#include <__type_traits/enable_if.h> +#include <__type_traits/integral_constant.h> +#include <__type_traits/is_constant_evaluated.h> +#include <__type_traits/is_nothrow_constructible.h> +#include <__type_traits/is_nothrow_destructible.h> +#include <__type_traits/is_nothrow_relocatable.h> +#include <__type_traits/is_trivially_relocatable.h> +#include <__type_traits/negation.h> +#include <__utility/move.h> +#include <__utility/scope_guard.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +// +// This file implements allocator-aware companions to the various relocation facilities. Those are +// necessary to use from allocator-aware containers like std::vector. +// + +template +struct __allocator_has_trivial_move_construct : _Not<__has_construct<_Alloc, _Type*, _Type&&> > {}; + +template +struct __allocator_has_trivial_move_construct, _Type> : true_type {}; + +template +struct __allocator_has_trivial_destroy : _Not<__has_destroy<_Alloc, _Tp*> > {}; + +template +struct __allocator_has_trivial_destroy, _Up> : true_type {}; + +// __is_trivially_allocator_relocatable: +// +// A type is trivially allocator-relocatable if the allocator's move construction and destruction +// don't do anything beyond calling the type's move constructor and destructor, and if the type +// itself is trivially relocatable. +template +struct __is_trivially_allocator_relocatable + : integral_constant::value && + __allocator_has_trivial_destroy<_Alloc, _Tp>::value && + __libcpp_is_trivially_relocatable<_Tp>::value> {}; + +// __is_nothrow_allocator_relocatable: +// +// A type is nothrow allocator-relocatable if the allocator operations are trivial and the type is +// nothrow relocatable, or if it is trivially allocator-relocatable. +template +struct __is_nothrow_allocator_relocatable + : _BoolConstant<(__allocator_has_trivial_move_construct<_Alloc, _Tp>::value && + __allocator_has_trivial_destroy<_Alloc, _Tp>::value && __is_nothrow_relocatable<_Tp>::value) || + __is_trivially_allocator_relocatable<_Alloc, _Tp>::value> {}; + +// __allocator_relocate_at: +// +// Either perform relocation using the allocator's non-trivial operations, or forward to allocator +// unaware relocation (which may then be trivial or not). +template ::value && + __allocator_has_trivial_destroy<_Alloc, _Tp>::value, + int> = 0> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* +__allocator_relocate_at(_Alloc& __alloc, _Tp* __dest, _Tp* __source) + _NOEXCEPT_(__is_nothrow_allocator_relocatable<_Alloc, _Tp>::value) { + (void)__alloc; // ignore the allocator since its operations are trivial anyway + return std::__relocate_at(__dest, __source); +} + +template +struct __allocator_destroy_object { + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __allocator_destroy_object(_Alloc& __alloc, _Tp* __obj) + : __alloc_(__alloc), __obj_(__obj) {} + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() const { + allocator_traits<_Alloc>::destroy(__alloc_, __obj_); + } + _Alloc& __alloc_; + _Tp* __obj_; +}; + +template ::value && + __allocator_has_trivial_destroy<_Alloc, _Tp>::value), + int> = 0> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* +__allocator_relocate_at(_Alloc& __alloc, _Tp* __dest, _Tp* __source) + _NOEXCEPT_(__is_nothrow_allocator_relocatable<_Alloc, _Tp>::value) { + auto __guard = std::__make_scope_guard(__allocator_destroy_object<_Alloc, _Tp>(__alloc, __source)); + allocator_traits<_Alloc>::construct(__alloc, __dest, std::move(*__source)); + return __dest; +} + +// __uninitialized_allocator_relocate: +// +// Equivalent to __uninitialized_relocate, but uses the provided allocator's construct() and destroy() methods. +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _NothrowForwardIter __uninitialized_allocator_relocate( + _Alloc& __alloc, _ForwardIter __first, _ForwardIter __last, _NothrowForwardIter __result) { +#ifndef _LIBCPP_CXX03_LANG + if constexpr (__libcpp_is_contiguous_iterator<_ForwardIter>::value && + __libcpp_is_contiguous_iterator<_NothrowForwardIter>::value) { + _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES( + !std::__is_pointer_in_range(std::__to_address(__first), std::__to_address(__last), std::__to_address(__last)), + "uninitialized_allocator_relocate requires the result not to overlap with the input range"); + } +#endif + + using _ValueType = typename iterator_traits<_ForwardIter>::value_type; + if _LIBCPP_CONSTEXPR (__allocator_has_trivial_move_construct<_Alloc, _ValueType>::value && + __allocator_has_trivial_destroy<_Alloc, _ValueType>::value) { + (void)__alloc; // ignore the allocator + return std::__uninitialized_relocate(std::move(__first), std::move(__last), std::move(__result)); + } else { + auto const __first_result = __result; + try { + while (__first != __last) { + std::__allocator_relocate_at(__alloc, std::addressof(*__result), std::addressof(*__first)); + ++__first; + ++__result; + } + } catch (...) { + std::__allocator_destroy(__alloc, ++__first, __last); + std::__allocator_destroy(__alloc, __first_result, __result); + throw; + } + return __result; + } +} + +// __uninitialized_allocator_relocate_backward: +// +// Equivalent to __uninitialized_relocate_backward, but uses the provided allocator's construct() and destroy() methods. +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _NothrowBidirectionalIter +__uninitialized_allocator_relocate_backward( + _Alloc& __alloc, _BidirectionalIter __first, _BidirectionalIter __last, _NothrowBidirectionalIter __result_last) { +#ifndef _LIBCPP_CXX03_LANG + if constexpr (__libcpp_is_contiguous_iterator<_BidirectionalIter>::value && + __libcpp_is_contiguous_iterator<_NothrowBidirectionalIter>::value) { + _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES( + !std::__is_pointer_in_range( + std::__to_address(__first), std::__to_address(__last), std::__to_address(__result_last) - 1), + "uninitialized_allocator_relocate_backward requires the end of the result not to overlap with the input range"); + } +#endif + + using _ValueType = typename iterator_traits<_BidirectionalIter>::value_type; + if _LIBCPP_CONSTEXPR (__allocator_has_trivial_move_construct<_Alloc, _ValueType>::value && + __allocator_has_trivial_destroy<_Alloc, _ValueType>::value) { + (void)__alloc; // ignore the allocator + return std::__uninitialized_relocate_backward(std::move(__first), std::move(__last), std::move(__result_last)); + } else { + auto __result = __result_last; + try { + while (__last != __first) { + --__last; + --__result; + std::__allocator_relocate_at(__alloc, std::addressof(*__result), std::addressof(*__last)); + } + } catch (...) { + std::__allocator_destroy(__alloc, __first, __last); + std::__allocator_destroy(__alloc, __result, __result_last); + throw; + } + + return __result; + } +} + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___MEMORY_ALLOCATOR_RELOCATION_H diff --git a/libcxx/include/__memory/relocate_at.h b/libcxx/include/__memory/relocate_at.h new file mode 100644 index 0000000000000..0783b429f03a1 --- /dev/null +++ b/libcxx/include/__memory/relocate_at.h @@ -0,0 +1,95 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___MEMORY_RELOCATE_AT_H +#define _LIBCPP___MEMORY_RELOCATE_AT_H + +#include <__memory/allocator_traits.h> +#include <__memory/construct_at.h> +#include <__type_traits/enable_if.h> +#include <__type_traits/integral_constant.h> +#include <__type_traits/is_constant_evaluated.h> +#include <__type_traits/is_constructible.h> +#include <__type_traits/is_nothrow_relocatable.h> +#include <__type_traits/is_same.h> +#include <__type_traits/is_trivially_relocatable.h> +#include <__utility/move.h> +#include <__utility/scope_guard.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +_LIBCPP_BEGIN_NAMESPACE_STD + +template +struct __relocatable_from + : _BoolConstant<_IsSame<_Source, _Dest>::value && + (is_move_constructible<_Dest>::value || __libcpp_is_trivially_relocatable<_Dest>::value)> {}; + +template +struct __destroy_object { + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __destroy_object(_Tp* __obj) : __obj_(__obj) {} + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void operator()() const { std::__destroy_at(__obj_); } + _Tp* __obj_; +}; + +template +_LIBCPP_HIDE_FROM_ABI _Tp* __libcpp_builtin_trivially_relocate_at(_Tp* __source, _Tp* __dest) _NOEXCEPT { + static_assert(__libcpp_is_trivially_relocatable<_Tp>::value, ""); + // Casting to void* to suppress clang complaining that this is technically UB. + __builtin_memcpy(static_cast(__dest), __source, sizeof(_Tp)); + return __dest; +} + +template +_LIBCPP_HIDE_FROM_ABI _Tp* __libcpp_builtin_trivially_relocate_at(_Tp* __first, _Tp* __last, _Tp* __dest) _NOEXCEPT { + static_assert(__libcpp_is_trivially_relocatable<_Tp>::value, ""); + // Casting to void* to suppress clang complaining that this is technically UB. + __builtin_memmove(static_cast(__dest), __first, (__last - __first) * sizeof(_Tp)); + return __dest; +} + +template ::value, int> = 0, + __enable_if_t<__libcpp_is_trivially_relocatable<_Tp>::value && is_move_constructible<_Tp>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __relocate_at(_Tp* __dest, _Tp* __source) + _NOEXCEPT_(__is_nothrow_relocatable<_Tp>::value) { + if (__libcpp_is_constant_evaluated()) { + auto __guard = std::__make_scope_guard(__destroy_object<_Tp>(__source)); + return std::__construct_at(__dest, std::move(*__source)); + } else { + return std::__libcpp_builtin_trivially_relocate_at(__source, __dest); + } +} + +template ::value, int> = 0, + __enable_if_t<__libcpp_is_trivially_relocatable<_Tp>::value && !is_move_constructible<_Tp>::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __relocate_at(_Tp* __dest, _Tp* __source) + _NOEXCEPT_(__is_nothrow_relocatable<_Tp>::value) { + return std::__libcpp_builtin_trivially_relocate_at(__source, __dest); +} + +template ::value, int> = 0, + __enable_if_t::value, int> = 0> +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __relocate_at(_Tp* __dest, _Tp* __source) + _NOEXCEPT_(__is_nothrow_relocatable<_Tp>::value) { + auto __guard = std::__make_scope_guard(__destroy_object<_Tp>(__source)); + return std::__construct_at(__dest, std::move(*__source)); +} + +_LIBCPP_END_NAMESPACE_STD + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___MEMORY_RELOCATE_AT_H diff --git a/libcxx/include/__memory/uninitialized_algorithms.h b/libcxx/include/__memory/uninitialized_algorithms.h index e80236640052c..072c5d4ad890b 100644 --- a/libcxx/include/__memory/uninitialized_algorithms.h +++ b/libcxx/include/__memory/uninitialized_algorithms.h @@ -20,6 +20,7 @@ #include <__iterator/iterator_traits.h> #include <__iterator/reverse_iterator.h> #include <__memory/addressof.h> +#include <__memory/allocator_relocation.h> #include <__memory/allocator_traits.h> #include <__memory/construct_at.h> #include <__memory/destroy.h> @@ -596,7 +597,7 @@ inline const bool __allocator_has_trivial_destroy_v = !__has_destroy_v<_Alloc, _ template inline const bool __allocator_has_trivial_destroy_v, _Up> = true; -// __uninitialized_allocator_relocate relocates the objects in [__first, __last) into __result. +// __uninitialized_allocator_relocate_strong relocates the objects in [__first, __last) into __result. // Relocation means that the objects in [__first, __last) are placed into __result as-if by move-construct and destroy, // except that the move constructor and destructor may never be called if they are known to be equivalent to a memcpy. // @@ -608,8 +609,12 @@ inline const bool __allocator_has_trivial_destroy_v, _Up> = true; // - is_nothrow_move_constructible<_ValueType> // - is_copy_constructible<_ValueType> // - __libcpp_is_trivially_relocatable<_ValueType> +// +// TODO: Replace this function by appropriate uses of __uninitialized_allocator_relocate from std::vector by +// handling the strong exception safety guarantee from outside the algorithm. The strong exception safety +// is a property of the data structure operation, not of this algorithm, so it doesn't belong here. template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __uninitialized_allocator_relocate( +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __uninitialized_allocator_relocate_strong( _Alloc& __alloc, _ContiguousIterator __first, _ContiguousIterator __last, _ContiguousIterator __result) { static_assert(__libcpp_is_contiguous_iterator<_ContiguousIterator>::value, ""); using _ValueType = typename iterator_traits<_ContiguousIterator>::value_type; diff --git a/libcxx/include/__memory/uninitialized_relocate.h b/libcxx/include/__memory/uninitialized_relocate.h new file mode 100644 index 0000000000000..97d2b1666ec02 --- /dev/null +++ b/libcxx/include/__memory/uninitialized_relocate.h @@ -0,0 +1,147 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H +#define _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H + +#include <__config> +#include <__iterator/iterator_traits.h> +#include <__memory/addressof.h> +#include <__memory/destroy.h> +#include <__memory/pointer_traits.h> +#include <__memory/relocate_at.h> +#include <__type_traits/is_constant_evaluated.h> +#include <__type_traits/is_trivially_relocatable.h> +#include <__utility/is_pointer_in_range.h> +#include <__utility/move.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_PUSH_MACROS +#include <__undef_macros> + +_LIBCPP_BEGIN_NAMESPACE_STD + +// __uninitialized_relocate relocates the objects in [__first, __last) into __result. +// +// Relocation means that the objects in [__first, __last) are placed into __result as-if by move-construct and destroy, +// except that the move constructor and destructor may never be called if they are known to be trivially relocatable. +// +// This algorithm works even if part of the resulting range overlaps with [__first, __last), as long as __result itself +// is not in [__first, last). +// +// If an exception is thrown, all the elements in the input range and in the output range are destroyed. +// +// Preconditions: +// - __result doesn't contain any objects and [__first, __last) contains objects +// - __result is not in [__first, __last) +// Postconditions: +// - __result contains the objects from [__first, __last) +// - [__first, __last) doesn't contain any objects +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _NothrowForwardIter +__uninitialized_relocate(_ForwardIter __first, _ForwardIter __last, _NothrowForwardIter __result) { +#ifndef _LIBCPP_CXX03_LANG + if constexpr (__libcpp_is_contiguous_iterator<_ForwardIter>::value && + __libcpp_is_contiguous_iterator<_NothrowForwardIter>::value) { + _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES( + !std::__is_pointer_in_range(std::__to_address(__first), std::__to_address(__last), std::__to_address(__result)), + "uninitialized_relocate requires the start of the result not to overlap with the input range"); + } + + // If we have contiguous iterators over a trivially relocatable type, use the builtin that is + // roughly equivalent to memmove. + using _ValueType = typename iterator_traits<_ForwardIter>::value_type; + if constexpr (__libcpp_is_contiguous_iterator<_ForwardIter>::value && + __libcpp_is_contiguous_iterator<_NothrowForwardIter>::value && + __libcpp_is_trivially_relocatable<_ValueType>::value) { + if (!__libcpp_is_constant_evaluated()) { + // TODO: We might be able to memcpy if we don't overlap at all? + std::__libcpp_builtin_trivially_relocate_at( + std::__to_address(__first), std::__to_address(__last), std::__to_address(__result)); + return __result + (__last - __first); + } + } +#endif + + // Otherwise, relocate elements one by one. + // + // If this throws an exception, we destroy the rest of the range we were relocating, and + // we also destroy everything we had relocated up to now. + auto const __first_result = __result; + try { + while (__first != __last) { + std::__relocate_at(std::addressof(*__result), std::addressof(*__first)); + ++__first; + ++__result; + } + } catch (...) { + std::__destroy(++__first, __last); + std::__destroy(__first_result, __result); + throw; + } + return __result; +} + +// __uninitialized_relocate_backward is like __uninitialized_relocate, but it relocates the elements in +// the range [first, last) to another range ending at __result_last. The elements are relocated in reverse +// order, but their relative order is preserved. +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _NothrowBidirectionalIter __uninitialized_relocate_backward( + _BidirectionalIter __first, _BidirectionalIter __last, _NothrowBidirectionalIter __result_last) { +#ifndef _LIBCPP_CXX03_LANG + if constexpr (__libcpp_is_contiguous_iterator<_BidirectionalIter>::value && + __libcpp_is_contiguous_iterator<_NothrowBidirectionalIter>::value) { + _LIBCPP_ASSERT_NON_OVERLAPPING_RANGES( + !std::__is_pointer_in_range( + std::__to_address(__first), std::__to_address(__last), std::__to_address(__result_last) - 1), + "uninitialized_relocate_backward requires the end of the result not to overlap with the input range"); + } + + // If we have contiguous iterators over a trivially relocatable type, use the builtin that is + // roughly equivalent to memmove. + using _ValueType = typename iterator_traits<_BidirectionalIter>::value_type; + if constexpr (__libcpp_is_contiguous_iterator<_BidirectionalIter>::value && + __libcpp_is_contiguous_iterator<_NothrowBidirectionalIter>::value && + __libcpp_is_trivially_relocatable<_ValueType>::value) { + if (!__libcpp_is_constant_evaluated()) { + auto __result = __result_last - (__last - __first); + // TODO: We might be able to memcpy if we don't overlap at all? + std::__libcpp_builtin_trivially_relocate_at( + std::__to_address(__first), std::__to_address(__last), std::__to_address(__result)); + return __result; + } + } +#endif + + // Otherwise, relocate elements one by one, starting from the end. + // + // If this throws an exception, we destroy the rest of the range we were relocating, and + // we also destroy everything we had relocated up to now. + auto __result = __result_last; + try { + while (__last != __first) { + --__last; + --__result; + std::__relocate_at(std::addressof(*__result), std::addressof(*__last)); + } + } catch (...) { + std::__destroy(__first, __last); + std::__destroy(__result, __result_last); + throw; + } + return __result; +} + +_LIBCPP_END_NAMESPACE_STD + +_LIBCPP_POP_MACROS + +#endif // _LIBCPP___MEMORY_UNINITIALIZED_RELOCATE_H diff --git a/libcxx/include/__type_traits/is_nothrow_relocatable.h b/libcxx/include/__type_traits/is_nothrow_relocatable.h new file mode 100644 index 0000000000000..b1055fdfe88a1 --- /dev/null +++ b/libcxx/include/__type_traits/is_nothrow_relocatable.h @@ -0,0 +1,33 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef _LIBCPP___TYPE_TRAITS_IS_NOTHROW_RELOCATABLE_H +#define _LIBCPP___TYPE_TRAITS_IS_NOTHROW_RELOCATABLE_H + +#include <__config> +#include <__type_traits/is_nothrow_constructible.h> +#include <__type_traits/is_nothrow_destructible.h> +#include <__type_traits/is_trivially_relocatable.h> +#include <__type_traits/remove_all_extents.h> + +#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) +# pragma GCC system_header +#endif + +_LIBCPP_BEGIN_NAMESPACE_STD + +template +struct __is_nothrow_relocatable + : integral_constant::value || + (is_nothrow_move_constructible<__remove_all_extents_t<_Tp> >::value && + is_nothrow_destructible<__remove_all_extents_t<_Tp> >::value)> {}; + +_LIBCPP_END_NAMESPACE_STD + +#endif // _LIBCPP___TYPE_TRAITS_IS_NOTHROW_RELOCATABLE_H diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h index 4307e78f6ddbc..1b6e60507f0f9 100644 --- a/libcxx/include/__vector/vector.h +++ b/libcxx/include/__vector/vector.h @@ -851,7 +851,7 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer& __v) { __annotate_delete(); auto __new_begin = __v.__begin_ - (__end_ - __begin_); - std::__uninitialized_allocator_relocate( + std::__uninitialized_allocator_relocate_strong( this->__alloc_, std::__to_address(__begin_), std::__to_address(__end_), std::__to_address(__new_begin)); __v.__begin_ = __new_begin; __end_ = __begin_; // All the objects have been destroyed by relocating them. @@ -874,13 +874,13 @@ vector<_Tp, _Allocator>::__swap_out_circular_buffer(__split_buffer__alloc_, std::__to_address(__p), std::__to_address(__end_), std::__to_address(__v.__end_)); __v.__end_ += (__end_ - __p); __end_ = __p; // The objects in [__p, __end_) have been destroyed by relocating them. auto __new_begin = __v.__begin_ - (__p - __begin_); - std::__uninitialized_allocator_relocate( + std::__uninitialized_allocator_relocate_strong( this->__alloc_, std::__to_address(__begin_), std::__to_address(__p), std::__to_address(__new_begin)); __v.__begin_ = __new_begin; __end_ = __begin_; // All the objects have been destroyed by relocating them. @@ -1311,12 +1311,12 @@ vector<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __position, _Inpu __v.__construct_at_end_with_sentinel(std::move(__first), std::move(__last)); __split_buffer __merged( __recommend(size() + __v.size()), __off, __alloc_); // has `__off` positions available at the front - std::__uninitialized_allocator_relocate( + std::__uninitialized_allocator_relocate_strong( __alloc_, std::__to_address(__old_last), std::__to_address(this->__end_), std::__to_address(__merged.__end_)); __guard.__complete(); // Release the guard once objects in [__old_last_, __end_) have been successfully relocated. __merged.__end_ += this->__end_ - __old_last; this->__end_ = __old_last; - std::__uninitialized_allocator_relocate( + std::__uninitialized_allocator_relocate_strong( __alloc_, std::__to_address(__v.__begin_), std::__to_address(__v.__end_), std::__to_address(__merged.__end_)); __merged.__end_ += __v.size(); __v.__end_ = __v.__begin_; diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in index 1d2d51275704b..fc42c19b5b345 100644 --- a/libcxx/include/module.modulemap.in +++ b/libcxx/include/module.modulemap.in @@ -235,6 +235,10 @@ module std_core [system] { header "__type_traits/is_nothrow_destructible.h" export std_core.type_traits.integral_constant } + module is_nothrow_relocatable { + header "__type_traits/is_nothrow_relocatable.h" + export std_core.type_traits.integral_constant + } module is_null_pointer { header "__type_traits/is_null_pointer.h" export std_core.type_traits.integral_constant @@ -1641,6 +1645,7 @@ module std [system] { } module allocator_arg_t { header "__memory/allocator_arg_t.h" } module allocator_destructor { header "__memory/allocator_destructor.h" } + module allocator_relocation { header "__memory/allocator_relocation.h" } module allocator_traits { header "__memory/allocator_traits.h" } module array_cookie { header "__memory/array_cookie.h" } module assume_aligned { header "__memory/assume_aligned.h" } @@ -1663,6 +1668,7 @@ module std [system] { export std.algorithm.in_out_result } module raw_storage_iterator { header "__memory/raw_storage_iterator.h" } + module relocate_at { header "__memory/relocate_at.h" } module shared_count { header "__memory/shared_count.h" } module shared_ptr { header "__memory/shared_ptr.h" } module swap_allocator { header "__memory/swap_allocator.h" } @@ -1675,6 +1681,9 @@ module std [system] { header "__memory/uninitialized_algorithms.h" export std.utility.pair } + module uninitialized_relocate { + header "__memory/uninitialized_relocate.h" + } module unique_ptr { header "__memory/unique_ptr.h" } diff --git a/libcxx/test/libcxx/memory/uninitialized_allocator_relocate.pass.cpp b/libcxx/test/libcxx/memory/uninitialized_allocator_relocate.pass.cpp new file mode 100644 index 0000000000000..2c4a15ad0436e --- /dev/null +++ b/libcxx/test/libcxx/memory/uninitialized_allocator_relocate.pass.cpp @@ -0,0 +1,284 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// Test the std::__uninitialized_allocator_relocate internal algorithm. + +// This test is impossible to write without std::to_address and std::construct_at +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +#include <__memory/allocator_relocation.h> +#include +#include +#include + +#include "min_allocator.h" +#include "test_iterators.h" +#include "test_macros.h" +#include "type_algorithms.h" + +template +struct Fixture { + using Traits = std::allocator_traits; + using ValueType = typename Traits::value_type; + using Pointer = typename Traits::pointer; + + constexpr Fixture(std::size_t n) : size(n) { + source = allocator.allocate(n); + for (std::size_t i = 0; i != n; ++i) { + Traits::construct(allocator, std::to_address(source + i), ValueType(i)); + } + + dest = allocator.allocate(n); + } + + constexpr void relocated(std::size_t n) { relocated_ = n; } + + constexpr ~Fixture() { + for (std::size_t i = 0; i != relocated_; ++i) { + Traits::destroy(allocator, std::to_address(dest + i)); + } + allocator.deallocate(dest, size); + + for (std::size_t i = relocated_; i != size; ++i) { + Traits::destroy(allocator, std::to_address(source + i)); + } + allocator.deallocate(source, size); + } + + Allocator allocator; + std::size_t size; + std::size_t relocated_ = 0; + Pointer source; + Pointer dest; +}; + +struct AliveTracker { + explicit AliveTracker(int v, bool do_throw, bool* alive) : value(v), do_throw_(do_throw), alive_(alive) { + if (alive_ != nullptr) + *alive_ = true; + } + AliveTracker(AliveTracker&& other) : value(other.value), do_throw_(other.do_throw_), alive_(other.alive_) { + other.alive_ = nullptr; + if (do_throw_) { + if (alive_ != nullptr) + *alive_ = false; // we failed to be constructed + throw 42; + } + } + ~AliveTracker() { + if (alive_ != nullptr) { + assert(*alive_ == true && "double destroy?"); + *alive_ = false; + } + } + int value; + bool do_throw_; + bool* alive_; +}; + +template