From 2355d8003b610c465b2ae3dd6338ed5d1fdd92df Mon Sep 17 00:00:00 2001 From: Nikolas Klauser Date: Mon, 15 Sep 2025 10:37:41 +0200 Subject: [PATCH] [libc++] Use relocation in vector::emplace_back --- libcxx/include/__utility/exception_guard.h | 5 +- libcxx/include/__vector/vector.h | 115 ++++++++++++--------- 2 files changed, 72 insertions(+), 48 deletions(-) diff --git a/libcxx/include/__utility/exception_guard.h b/libcxx/include/__utility/exception_guard.h index 6fa744e8b4f32..8d0da912f9850 100644 --- a/libcxx/include/__utility/exception_guard.h +++ b/libcxx/include/__utility/exception_guard.h @@ -80,7 +80,10 @@ struct __exception_guard_exceptions { _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __complete() _NOEXCEPT { __completed_ = true; } - _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__exception_guard_exceptions() { + // __exception_guard is almost always used with a lambda, so the destructor is almost certainly unique to the calling + // function. LLVM doesn't know that, so it doesn't inline it due to the destructor being in a cold (exception) path. + // Annotate the function with always_inline to work around those problems. + _LIBCPP_ALWAYS_INLINE _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__exception_guard_exceptions() { if (!__completed_) __rollback_(); } diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h index 27e681aeef22a..f2bca70e68cd7 100644 --- a/libcxx/include/__vector/vector.h +++ b/libcxx/include/__vector/vector.h @@ -84,6 +84,24 @@ _LIBCPP_PUSH_MACROS _LIBCPP_BEGIN_NAMESPACE_STD +// This makes the compiler inline `__else()` if `__cond` is known to be false. Currently LLVM doesn't do that without +// the `__builtin_constant_p`, since it considers `__else` unlikely even through it's known to be run. +// See https://llvm.org/PR154292 +template +_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __if_likely_else(bool __cond, _If __if, _Else __else) { + if (__builtin_constant_p(__cond)) { + if (__cond) + __if(); + else + __else(); + } else { + if (__cond) [[__likely__]] + __if(); + else + __else(); + } +} + template */> class vector { template @@ -469,15 +487,60 @@ class vector { _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x) { emplace_back(std::move(__x)); } template - _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 reference __emplace_back(_Args&&... __args) { + if constexpr (__libcpp_is_trivially_relocatable::value && + __allocator_has_trivial_move_construct_v && + __allocator_has_trivial_destroy_v) { + // This path is written in a way to have the fast path as compact as possible. Specifically, + // there is a branch in case there isn't enough capacity left over, which will return to the same location within + // the function after growing the vector. This ensures that the relocation code exists only once and the + // reallocation path is common across all `vector` instantiations of trivially relocatable types. + union _Tmp { + _LIBCPP_DIAGNOSTIC_PUSH + // Clang complains about __exclude_from_explicit_instantiation__ on a local class member. + _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wignored-attributes") + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tmp() {} + _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~_Tmp() {} + _LIBCPP_DIAGNOSTIC_POP + value_type __val_; + }; + _Tmp __tmp; + + __alloc_traits::construct(__alloc_, std::addressof(__tmp.__val_), std::forward<_Args>(__args)...); + + auto __guard = + std::__make_exception_guard([&, this] { __alloc_traits::destroy(__alloc_, std::addressof(__tmp.__val_)); }); + std::__if_likely_else(size() != capacity(), [] {}, [this] { reserve(__recommend(size() + 1)); }); + __guard.__complete(); + std::__uninitialized_allocator_relocate( + __alloc_, std::addressof(__tmp.__val_), std::addressof(__tmp.__val_) + 1, std::__to_address(__end_)); + ++__end_; + } else { + pointer __end = this->__end_; + std::__if_likely_else( + __end < this->__cap_, + [&] { + __emplace_back_assume_capacity(std::forward<_Args>(__args)...); + ++__end; + }, + [&] { __end = __emplace_back_slow_path(std::forward<_Args>(__args)...); }); + + this->__end_ = __end; + } + return __end_[-1]; + } + #if _LIBCPP_STD_VER >= 17 - reference - emplace_back(_Args&&... __args); + using __emplace_back_result = reference; #else - void - emplace_back(_Args&&... __args); + using __emplace_back_result = void; #endif + template + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __emplace_back_result emplace_back(_Args&&... __args) { + return static_cast<__emplace_back_result>(__emplace_back(std::forward<_Args>(__args)...)); + } + template _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __emplace_back_assume_capacity(_Args&&... __args) { _LIBCPP_ASSERT_INTERNAL( @@ -1161,48 +1224,6 @@ vector<_Tp, _Allocator>::__emplace_back_slow_path(_Args&&... __args) { return this->__end_; } -// This makes the compiler inline `__else()` if `__cond` is known to be false. Currently LLVM doesn't do that without -// the `__builtin_constant_p`, since it considers `__else` unlikely even through it's known to be run. -// See https://llvm.org/PR154292 -template -_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __if_likely_else(bool __cond, _If __if, _Else __else) { - if (__builtin_constant_p(__cond)) { - if (__cond) - __if(); - else - __else(); - } else { - if (__cond) [[__likely__]] - __if(); - else - __else(); - } -} - -template -template -_LIBCPP_CONSTEXPR_SINCE_CXX20 inline -#if _LIBCPP_STD_VER >= 17 - typename vector<_Tp, _Allocator>::reference -#else - void -#endif - vector<_Tp, _Allocator>::emplace_back(_Args&&... __args) { - pointer __end = this->__end_; - std::__if_likely_else( - __end < this->__cap_, - [&] { - __emplace_back_assume_capacity(std::forward<_Args>(__args)...); - ++__end; - }, - [&] { __end = __emplace_back_slow_path(std::forward<_Args>(__args)...); }); - - this->__end_ = __end; -#if _LIBCPP_STD_VER >= 17 - return *(__end - 1); -#endif -} - template _LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator vector<_Tp, _Allocator>::erase(const_iterator __position) {