Relocation algorithms Clean

libs/core/algorithms/CMakeLists.txt

@@ -97,6 +97,7 @@ set(algorithms_headers
     hpx/parallel/algorithms/uninitialized_default_construct.hpp
     hpx/parallel/algorithms/uninitialized_fill.hpp
     hpx/parallel/algorithms/uninitialized_move.hpp
+    hpx/parallel/algorithms/uninitialized_relocate.hpp
     hpx/parallel/algorithms/uninitialized_value_construct.hpp
     hpx/parallel/algorithms/unique.hpp
     hpx/parallel/container_algorithms/adjacent_difference.hpp

libs/core/algorithms/include/hpx/algorithms/traits/pointer_category.hpp

@@ -9,6 +9,7 @@
 
 #include <hpx/iterator_support/traits/is_iterator.hpp>
 #include <hpx/type_support/is_relocatable.hpp>
+#include <hpx/type_support/is_trivially_relocatable.hpp>
 
 #include <type_traits>
 
@@ -25,7 +26,11 @@ namespace hpx::traits {
     {
     };
 
-    struct relocatable_pointer_tag : general_pointer_tag
+    struct trivially_relocatable_pointer_tag : general_pointer_tag
+    {
+    };
+
+    struct nothrow_relocatable_pointer_tag : general_pointer_tag
     {
     };
 
@@ -139,20 +144,34 @@ namespace hpx::traits {
             using type = general_pointer_tag;
         };
 
-        template <typename Source, typename Dest,
-            bool Contiguous = iterators_are_contiguous_v<Source, Dest>>
+        template <typename Source, typename Dest>
         struct pointer_relocate_category
         {
-            using type = std::conditional_t<
-                std::is_same_v<iter_value_t<Source>, iter_value_t<Dest>> &&
-                    is_relocatable_v<iter_value_t<Source>>,
-                relocatable_pointer_tag, general_pointer_tag>;
-        };
+            using type_src = iter_value_t<Source>;
+            using type_dst = iter_value_t<Dest>;
 
-        template <typename Source, typename Dest>
-        struct pointer_relocate_category<Source, Dest, false>
-        {
-            using type = general_pointer_tag;
+            constexpr static bool valid_relocation =
+                is_relocatable_from_v<type_src, type_dst>;
+
+            constexpr static bool memcpy_legal =
+                !std::is_volatile_v<type_src> && !std::is_volatile_v<type_dst>;
+
+            constexpr static bool is_buffer_memcpyable = valid_relocation &&
+                is_trivially_relocatable_v<type_src> &&
+                iterators_are_contiguous_v<Source, Dest> && memcpy_legal;
+
+            constexpr static bool can_move_construct_nothrow =
+                std::is_nothrow_constructible_v<type_src,
+                    std::add_rvalue_reference_t<type_dst>>;
+
+            // Used externally to determine if a relocation is noexcept
+            constexpr static bool is_noexcept_relocatable_v =
+                can_move_construct_nothrow && is_buffer_memcpyable;
+
+            using type = std::conditional_t<is_buffer_memcpyable,
+                trivially_relocatable_pointer_tag,
+                std::conditional_t<can_move_construct_nothrow,
+                    nothrow_relocatable_pointer_tag, general_pointer_tag>>;
         };
     }    // namespace detail
 

libs/core/algorithms/include/hpx/parallel/util/loop.hpp

@@ -10,6 +10,7 @@
 #include <hpx/assert.hpp>
 #include <hpx/datastructures/tuple.hpp>
 #include <hpx/execution/traits/is_execution_policy.hpp>
+#include <hpx/executors/execution_policy.hpp>
 #include <hpx/functional/detail/invoke.hpp>
 #include <hpx/functional/detail/tag_fallback_invoke.hpp>
 #include <hpx/functional/invoke_result.hpp>
@@ -781,6 +782,176 @@ namespace hpx::parallel::util {
             dest, tok, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup));
     }
 
+    namespace detail {
+        // Helper class to repeatedly call a function a given number of times
+        // starting from a given iterator position. The cleanup function is
+        // called with an argument of the iterator that was being processed
+        // when the exception was thrown.
+        struct loop_with_manual_cleanup_n final
+          : hpx::functional::detail::tag_fallback<loop_with_manual_cleanup_n>
+        {
+            template <typename ExPolicy, typename FwdIter, typename F,
+                typename Cleanup>
+            static FwdIter call(ExPolicy&&, FwdIter it, std::size_t num, F&& f,
+                Cleanup&& cleanup)
+            {
+                try
+                {
+                    std::size_t count(num & std::size_t(-4));    // -V112
+                    for (std::size_t i = 0; i < count;
+                         (void) ++it, i += 4)    // -V112
+                    {
+                        HPX_INVOKE(f, it);
+                        HPX_INVOKE(f, ++it);
+                        HPX_INVOKE(f, ++it);
+                        HPX_INVOKE(f, ++it);
+                    }
+                    for (/**/; count < num; (void) ++count, ++it)
+                    {
+                        HPX_INVOKE(f, it);
+                    }
+                    return it;
+                }
+                catch (...)
+                {
+                    // Give the user the flexibility to
+                    // clean up whichever iterators they want
+                    // only providing the iterator that
+                    // was being processed when the exception
+                    // was thrown
+                    HPX_INVOKE(cleanup, it);
+                    throw;
+                }
+            }
+
+            // Manual cleanup
+            template <typename ExPolicy, typename FwdIter, typename F,
+                typename Cleanup>
+            friend HPX_HOST_DEVICE HPX_FORCEINLINE FwdIter tag_fallback_invoke(
+                loop_with_manual_cleanup_n, ExPolicy&& policy, FwdIter it,
+                std::size_t num, F&& f, Cleanup&& cleanup)
+            {
+                return call(HPX_FORWARD(ExPolicy, policy), it, num,
+                    HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup));
+            }
+
+            // Manual cleanup with token
+            template <typename ExPolicy, typename FwdIter, typename CancelToken,
+                typename F, typename Cleanup>
+            HPX_HOST_DEVICE HPX_FORCEINLINE static FwdIter call_with_token(
+                ExPolicy&& policy, FwdIter it, std::size_t num,
+                CancelToken& tok, F&& f, Cleanup&& cleanup)
+            {
+                // check at the start of a partition only
+                if (tok.was_cancelled())
+                    return it;
+
+                return call(HPX_FORWARD(ExPolicy, policy), it, num,
+                    HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup));
+            }
+
+            // Manual cleanup
+            template <typename ExPolicy, typename FwdIter, typename F,
+                typename Cleanup>
+            static FwdIter unseq_call(
+                ExPolicy, FwdIter it, std::size_t num, F&& f, Cleanup&& cleanup)
+            {
+                try
+                {
+                    std::size_t count(num & std::size_t(-4));    // -V112
+
+                    // clang-format off
+                    HPX_IVDEP HPX_UNROLL HPX_VECTORIZE
+                    for (std::size_t i = 0; i < count;
+                         (void) ++it, ++i)    // -V112
+                    {
+                        HPX_INVOKE(f, it);
+                    }
+
+                    HPX_IVDEP HPX_UNROLL HPX_VECTORIZE
+                    for (/**/; count < num; (void) ++count, ++it)
+                    {
+                        HPX_INVOKE(f, it);
+                    }
+                    // clang-format on
+
+                    return it;
+                }
+                catch (...)
+                {
+                    // Give the user the flexibility to
+                    // clean up whichever iterators they want
+                    // only providing the iterator that
+                    // was being processed when the exception
+                    // was thrown
+                    HPX_INVOKE(cleanup, it);
+                    throw;
+                }
+            }
+
+        };    // struct loop_with_manual_cleanup_n
+
+    };    // namespace detail
+
+#if !defined(HPX_COMPUTE_DEVICE_CODE)
+    inline constexpr detail::loop_with_manual_cleanup_n
+        loop_with_manual_cleanup_n = detail::loop_with_manual_cleanup_n{};
+#else
+    template <typename ExPolicy, typename Begin, typename F, typename Cleanup>
+    HPX_HOST_DEVICE HPX_FORCEINLINE constexpr Begin loop_with_manual_cleanup_n(
+        ExPolicy&& policy, Begin begin, std::size_t n, F&& f, Cleanup&& cleanup)
+    {
+        return hpx::parallel::util::detail::loop_with_manual_cleanup_n{}(
+            HPX_FORWARD(ExPolicy, policy), begin, n, HPX_FORWARD(F, f),
+            HPX_FORWARD(Cleanup, cleanup));
+    }
+#endif
+
+    template <typename Begin, typename F, typename Cleanup>
+    HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke(
+        hpx::parallel::util::detail::loop_with_manual_cleanup_n,
+        hpx::execution::unsequenced_policy, Begin HPX_RESTRICT begin,
+        std::size_t n, F&& f, Cleanup&& cleanup)
+    {
+        return hpx::parallel::util::detail::loop_with_manual_cleanup_n::
+            unseq_call(
+                begin, n, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup));
+    }
+
+    template <typename Begin, typename F, typename Cleanup>
+    HPX_HOST_DEVICE HPX_FORCEINLINE Begin tag_invoke(
+        hpx::parallel::util::detail::loop_with_manual_cleanup_n,
+        hpx::execution::unsequenced_task_policy, Begin HPX_RESTRICT begin,
+        std::size_t n, F&& f, Cleanup&& cleanup)
+    {
+        return hpx::parallel::util::detail::loop_with_manual_cleanup_n::
+            unseq_call(
+                begin, n, HPX_FORWARD(F, f), HPX_FORWARD(Cleanup, cleanup));
+    }
+
+    template <typename Iter, typename CancelToken, typename F, typename Cleanup,
+        HPX_CONCEPT_REQUIRES_(hpx::traits::is_iterator_v<Iter>)>
+    HPX_FORCEINLINE constexpr Iter loop_with_manual_cleanup_n_with_token(
+        Iter it, std::size_t count, CancelToken& tok, F&& f, Cleanup&& cleanup)
+    {
+        return detail::loop_with_manual_cleanup_n::call_with_token(
+            ::hpx::execution::seq, it, count, tok, HPX_FORWARD(F, f),
+            HPX_FORWARD(Cleanup, cleanup));
+    }
+
+    template <typename ExPolicy, typename Iter, typename CancelToken,
+        typename F, typename Cleanup,
+        HPX_CONCEPT_REQUIRES_(hpx::is_execution_policy_v<ExPolicy>&&
+                hpx::traits::is_iterator_v<Iter>)>    // could be a zip iterator
+    HPX_FORCEINLINE constexpr Iter loop_with_manual_cleanup_n_with_token(
+        ExPolicy&& policy, Iter it, std::size_t count, CancelToken& tok, F&& f,
+        Cleanup&& cleanup)
+    {
+        return detail::loop_with_manual_cleanup_n::call_with_token(
+            HPX_FORWARD(ExPolicy, policy), it, count, tok, HPX_FORWARD(F, f),
+            HPX_FORWARD(Cleanup, cleanup));
+    }
+
     ///////////////////////////////////////////////////////////////////////////
     namespace detail {