diff --git a/libcxx/include/__vector/vector.h b/libcxx/include/__vector/vector.h index 5e9fa4a7d0030..50e3e9dab0cee 100644 --- a/libcxx/include/__vector/vector.h +++ b/libcxx/include/__vector/vector.h @@ -58,6 +58,7 @@ #include <__type_traits/is_same.h> #include <__type_traits/is_swappable.h> #include <__type_traits/is_trivially_relocatable.h> +#include <__type_traits/remove_const_ref.h> #include <__type_traits/type_identity.h> #include <__utility/declval.h> #include <__utility/exception_guard.h> @@ -504,9 +505,14 @@ class vector { this->__destruct_at_end(__layout_.__end_ptr() - 1); } - _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, const_reference __x); + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, const_reference __x) { + return emplace(__position, __x); + } + + _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, value_type&& __x) { + return emplace(__position, std::move(__x)); + } - _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, value_type&& __x); template _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __position, _Args&&... __args); @@ -1128,49 +1134,6 @@ vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointe std::move_backward(__from_s, __from_s + __n, __old_last); } -template -_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator -vector<_Tp, _Allocator>::insert(const_iterator __position, const_reference __x) { - pointer __p = this->__layout_.__begin_ptr() + (__position - begin()); - if (size() != capacity()) { - pointer __end = __layout_.__end_ptr(); - if (__p == __end) { - __emplace_back_assume_capacity(__x); - } else { - __move_range(__p, __end, __p + 1); - const_pointer __xr = pointer_traits::pointer_to(__x); - if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end), std::addressof(__x))) - ++__xr; - *__p = *__xr; - } - } else { - _SplitBuffer __v(__recommend(size() + 1), __p - this->__layout_.__begin_ptr(), this->__layout_.__alloc()); - __v.emplace_back(__x); - __p = __layout_.__relocate_with_pivot(__v, __p); - } - return __make_iter(__p); -} - -template -_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator -vector<_Tp, _Allocator>::insert(const_iterator __position, value_type&& __x) { - pointer __p = this->__layout_.__begin_ptr() + (__position - begin()); - if (size() != capacity()) { - pointer __end = __layout_.__end_ptr(); - if (__p == __end) { - __emplace_back_assume_capacity(std::move(__x)); - } else { - __move_range(__p, __end, __p + 1); - *__p = std::move(__x); - } - } else { - _SplitBuffer __v(__recommend(size() + 1), __p - this->__layout_.__begin_ptr(), this->__layout_.__alloc()); - __v.emplace_back(std::move(__x)); - __p = __layout_.__relocate_with_pivot(__v, __p); - } - return __make_iter(__p); -} - template template _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator @@ -1181,6 +1144,19 @@ vector<_Tp, _Allocator>::emplace(const_iterator __position, _Args&&... __args) { if (__p == __end) { __emplace_back_assume_capacity(std::forward<_Args>(__args)...); } else { +#ifndef _LIBCPP_CXX03_LANG + // If we insert a single element that is the same as the value_type, avoid creating a temporary. + if constexpr (sizeof...(_Args) == 1) { + if constexpr (is_same<__remove_const_ref_t<_Args>..., value_type>::value) { + __move_range(__p, __end, __p + 1); + auto __xr = std::addressof(__args...); + if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end), std::addressof(__args)...)) + ++__xr; + *__p = (std::forward<_Args>(*__xr), ...); + return __make_iter(__p); + } + } +#endif __temp_value __tmp(this->__layout_.__alloc(), std::forward<_Args>(__args)...); __move_range(__p, __end, __p + 1); *__p = std::move(__tmp.get()); diff --git a/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h b/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h index 5210008989ff7..17579c24723af 100644 --- a/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h +++ b/libcxx/test/benchmarks/containers/sequence/sequence_container_benchmarks.h @@ -184,7 +184,7 @@ void sequence_container_benchmarks(std::string container) { ///////////////////////// // Insertion ///////////////////////// - for (auto gen : generators) + for (auto gen : generators) { bench("insert(begin)" + tostr(gen), [gen](auto& st) { auto const size = st.range(0); std::vector in; @@ -205,8 +205,29 @@ void sequence_container_benchmarks(std::string container) { } }); + bench("emplace(begin)" + tostr(gen), [gen](auto& st) { + auto const size = st.range(0); + std::vector in; + std::generate_n(std::back_inserter(in), size, gen); + DoNotOptimizeData(in); + + Container c(in.begin(), in.end()); + DoNotOptimizeData(c); + + ValueType value = gen(); + benchmark::DoNotOptimize(value); + + for ([[maybe_unused]] auto _ : st) { + c.emplace(c.begin(), value); + DoNotOptimizeData(c); + + c.erase(std::prev(c.end())); // avoid growing indefinitely + } + }); + } + if constexpr (std::random_access_iterator) { - for (auto gen : generators) + for (auto gen : generators) { bench("insert(middle)" + tostr(gen), [gen](auto& st) { auto const size = st.range(0); std::vector in; @@ -227,6 +248,27 @@ void sequence_container_benchmarks(std::string container) { c.erase(c.end() - 1); // avoid growing indefinitely } }); + bench("emplace(middle)" + tostr(gen), [gen](auto& st) { + auto const size = st.range(0); + std::vector in; + std::generate_n(std::back_inserter(in), size, gen); + DoNotOptimizeData(in); + + Container c(in.begin(), in.end()); + DoNotOptimizeData(c); + + ValueType value = gen(); + benchmark::DoNotOptimize(value); + + for ([[maybe_unused]] auto _ : st) { + auto mid = c.begin() + (size / 2); // requires random-access iterators in order to make sense + c.emplace(mid, value); + DoNotOptimizeData(c); + + c.erase(c.end() - 1); // avoid growing indefinitely + } + }); + } } if constexpr (requires(Container c) { c.reserve(0); }) { @@ -319,7 +361,7 @@ void sequence_container_benchmarks(std::string container) { if constexpr (has_capacity) { // For containers where we can observe capacity(), push_back a single element // without reserving to ensure the container needs to grow - for (auto gen : generators) + for (auto gen : generators) { bench("push_back() (growing)" + tostr(gen), [gen](auto& st) { auto const size = st.range(0); std::vector in; @@ -349,12 +391,42 @@ void sequence_container_benchmarks(std::string container) { st.ResumeTiming(); } }); + bench("emplace_back() (growing)" + tostr(gen), [gen](auto& st) { + auto const size = st.range(0); + std::vector in; + std::generate_n(std::back_inserter(in), size, gen); + DoNotOptimizeData(in); + + auto at_capacity = [](Container c) { + while (c.size() < c.capacity()) + c.push_back(c.back()); + return c; + }; + + std::vector c(BatchSize, at_capacity(Container(in.begin(), in.end()))); + std::vector const original = c; + + while (st.KeepRunningBatch(BatchSize)) { + for (std::size_t i = 0; i != BatchSize; ++i) { + c[i].emplace_back(in[i]); + DoNotOptimizeData(c[i]); + } + + st.PauseTiming(); + for (std::size_t i = 0; i != BatchSize; ++i) { + c[i] = at_capacity(Container(in.begin(), in.end())); + assert(c[i].size() == c[i].capacity()); + } + st.ResumeTiming(); + } + }); + } } // For containers where we can reserve, push_back a single element after reserving to // ensure the container doesn't grow if constexpr (has_reserve) { - for (auto gen : generators) + for (auto gen : generators) { bench("push_back() (with reserve)" + tostr(gen), [gen](auto& st) { auto const size = st.range(0); std::vector in; @@ -375,10 +447,34 @@ void sequence_container_benchmarks(std::string container) { c.erase(c.end() - BatchSize, c.end()); } }); + + bench("emplace_back() (with reserve)" + tostr(gen), [gen](auto& st) { + auto const size = st.range(0); + std::vector in; + std::generate_n(std::back_inserter(in), size, gen); + DoNotOptimizeData(in); + + Container c(in.begin(), in.end()); + // Ensure the container has enough capacity + c.reserve(c.size() + BatchSize); + DoNotOptimizeData(c); + + while (st.KeepRunningBatch(BatchSize)) { + for (std::size_t i = 0; i != BatchSize; ++i) { + c.emplace_back(in[i]); + } + DoNotOptimizeData(c); + + st.PauseTiming(); + c.erase(c.end() - BatchSize, c.end()); + st.ResumeTiming(); + } + }); + } } // push_back many elements: this is amortized constant for std::vector but not all containers - for (auto gen : generators) + for (auto gen : generators) { bench("push_back() (many elements)" + tostr(gen), [gen](auto& st) { auto const size = st.range(0); std::vector in; @@ -398,6 +494,26 @@ void sequence_container_benchmarks(std::string container) { st.ResumeTiming(); } }); + bench("emplace_back() (many elements)" + tostr(gen), [gen](auto& st) { + auto const size = st.range(0); + std::vector in; + std::generate_n(std::back_inserter(in), size, gen); + DoNotOptimizeData(in); + + Container c; + DoNotOptimizeData(c); + while (st.KeepRunningBatch(size)) { + for (int i = 0; i != size; ++i) { + c.emplace_back(in[i]); + } + DoNotOptimizeData(c); + + st.PauseTiming(); + c.clear(); + st.ResumeTiming(); + } + }); + } #if TEST_STD_VER >= 23 for (auto gen : generators)