[libc++] Introduce __specialized_algorithms

libcxx/include/CMakeLists.txt

@@ -194,6 +194,7 @@ set(files
   __algorithm/simd_utils.h
   __algorithm/sort.h
   __algorithm/sort_heap.h
+  __algorithm/specialized_algorithms.h
   __algorithm/stable_partition.h
   __algorithm/stable_sort.h
   __algorithm/swap_ranges.h

libcxx/include/__algorithm/fill_n.h

@@ -10,13 +10,13 @@
 #define _LIBCPP___ALGORITHM_FILL_N_H
 
 #include <__algorithm/for_each_n_segment.h>
-#include <__algorithm/min.h>
+#include <__algorithm/specialized_algorithms.h>
 #include <__config>
-#include <__fwd/bit_reference.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/segmented_iterator.h>
-#include <__memory/pointer_traits.h>
+#include <__type_traits/enable_if.h>
 #include <__utility/convert_to_integral.h>
+#include <__utility/move.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -29,7 +29,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // fill_n isn't specialized for std::memset, because the compiler already optimizes the loop to a call to std::memset.
 
-template <class _OutputIterator, class _Size, class _Tp>
+template <
+    class _OutputIterator,
+    class _Size,
+    class _Tp,
+    __enable_if_t<!__specialized_algorithm<_Algorithm::__fill_n, __single_iterator<_OutputIterator> >::__has_algorithm,
+                  int> = 0>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutputIterator
 __fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) {
 #ifndef _LIBCPP_CXX03_LANG
@@ -47,42 +52,14 @@ __fill_n(_OutputIterator __first, _Size __n, const _Tp& __value) {
   return __first;
 }
 
-template <bool _FillVal, class _Cp>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
-__fill_n_bool(__bit_iterator<_Cp, false> __first, typename __size_difference_type_traits<_Cp>::size_type __n) {
-  using _It            = __bit_iterator<_Cp, false>;
-  using __storage_type = typename _It::__storage_type;
-
-  const int __bits_per_word = _It::__bits_per_word;
-  // do first partial word
-  if (__first.__ctz_ != 0) {
-    __storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
-    __storage_type __dn    = std::min(__clz_f, __n);
-    std::__fill_masked_range(std::__to_address(__first.__seg_), __clz_f - __dn, __first.__ctz_, _FillVal);
-    __n -= __dn;
-    ++__first.__seg_;
-  }
-  // do middle whole words
-  __storage_type __nw = __n / __bits_per_word;
-  std::__fill_n(std::__to_address(__first.__seg_), __nw, _FillVal ? static_cast<__storage_type>(-1) : 0);
-  __n -= __nw * __bits_per_word;
-  // do last partial word
-  if (__n > 0) {
-    __first.__seg_ += __nw;
-    std::__fill_masked_range(std::__to_address(__first.__seg_), __bits_per_word - __n, 0u, _FillVal);
-  }
-}
-
-template <class _Cp, class _Size>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 __bit_iterator<_Cp, false>
-__fill_n(__bit_iterator<_Cp, false> __first, _Size __n, const bool& __value) {
-  if (__n > 0) {
-    if (__value)
-      std::__fill_n_bool<true>(__first, __n);
-    else
-      std::__fill_n_bool<false>(__first, __n);
-  }
-  return __first + __n;
+template <class _OutIter,
+          class _Size,
+          class _Tp,
+          __enable_if_t<__specialized_algorithm<_Algorithm::__fill_n, __single_iterator<_OutIter> >::__has_algorithm,
+                        int> = 0>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _OutIter __fill_n(_OutIter __first, _Size __n, const _Tp& __value) {
+  return __specialized_algorithm<_Algorithm::__fill_n, __single_iterator<_OutIter> >()(
+      std::move(__first), __n, __value);
 }
 
 template <class _OutputIterator, class _Size, class _Tp>

libcxx/include/__algorithm/specialized_algorithms.h

@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___ALGORITHM_SPECIALIZED_ALGORITHMS_H
+#define _LIBCPP___ALGORITHM_SPECIALIZED_ALGORITHMS_H
+
+#include <__config>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// FIXME: This should really be an enum
+namespace _Algorithm {
+  struct __fill_n {};
+} // namespace _Algorithm
+
+template <class>
+struct __single_iterator;
+
+template <class _Alg, class... _Ranges>
+struct __specialized_algorithm {
+  static const bool __has_algorithm = false;
+};
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___ALGORITHM_SPECIALIZED_ALGORITHMS_H

libcxx/include/__bit_reference

@@ -15,8 +15,10 @@
 #include <__algorithm/copy_backward.h>
 #include <__algorithm/copy_n.h>
 #include <__algorithm/equal.h>
+#include <__algorithm/fill_n.h>
 #include <__algorithm/min.h>
 #include <__algorithm/rotate.h>
+#include <__algorithm/specialized_algorithms.h>
 #include <__algorithm/swap_ranges.h>
 #include <__assert>
 #include <__bit/countr.h>
@@ -531,12 +533,55 @@ private:
       _Pred&,
       _Proj1&,
       _Proj2&);
-  template <bool _ToFind, class _Dp, bool _IC>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, _IC>
-      __find_bool(__bit_iterator<_Dp, _IC>, typename __size_difference_type_traits<_Dp>::size_type);
   template <bool _ToCount, class _Dp, bool _IC>
   friend typename __bit_iterator<_Dp, _IC>::difference_type _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
   __count_bool(__bit_iterator<_Dp, _IC>, typename __size_difference_type_traits<_Dp>::size_type);
+
+  template <class, class...>
+  friend struct __specialized_algorithm;
+};
+
+template <class _Cp>
+struct __specialized_algorithm<_Algorithm::__fill_n, __single_iterator<__bit_iterator<_Cp, false> > > {
+  static const bool __has_algorithm = true;
+
+  template <bool _FillVal>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void
+  __impl(__bit_iterator<_Cp, false> __first, typename __size_difference_type_traits<_Cp>::size_type __n) {
+    using _It            = __bit_iterator<_Cp, false>;
+    using __storage_type = typename _It::__storage_type;
+
+    const int __bits_per_word = _It::__bits_per_word;
+    // do first partial word
+    if (__first.__ctz_ != 0) {
+      __storage_type __clz_f = static_cast<__storage_type>(__bits_per_word - __first.__ctz_);
+      __storage_type __dn    = std::min(__clz_f, __n);
+      std::__fill_masked_range(std::__to_address(__first.__seg_), __clz_f - __dn, __first.__ctz_, _FillVal);
+      __n -= __dn;
+      ++__first.__seg_;
+    }
+    // do middle whole words
+    __storage_type __nw = __n / __bits_per_word;
+    std::__fill_n(std::__to_address(__first.__seg_), __nw, _FillVal ? static_cast<__storage_type>(-1) : 0);
+    __n -= __nw * __bits_per_word;
+    // do last partial word
+    if (__n > 0) {
+      __first.__seg_ += __nw;
+      std::__fill_masked_range(std::__to_address(__first.__seg_), __bits_per_word - __n, 0u, _FillVal);
+    }
+  }
+
+  template <class _Size, class _Tp>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static __bit_iterator<_Cp, false>
+  operator()(__bit_iterator<_Cp, false> __first, _Size __n, const _Tp& __value) {
+    if (__n > 0) {
+      if (__value)
+        __impl<true>(__first, __n);
+      else
+        __impl<false>(__first, __n);
+    }
+    return __first + __n;
+  }
 };
 
 _LIBCPP_END_NAMESPACE_STD

libcxx/include/module.modulemap.in

@@ -839,6 +839,7 @@ module std [system] {
     module simd_utils                             { header "__algorithm/simd_utils.h" }
     module sort_heap                              { header "__algorithm/sort_heap.h" }
     module sort                                   { header "__algorithm/sort.h" }
+    module specialized_algorithms                 { header "__algorithm/specialized_algorithms.h" }
     module stable_partition                       { header "__algorithm/stable_partition.h" }
     module stable_sort {
       header "__algorithm/stable_sort.h"