Merge bdfd6d4 into dd17010

.jenkins/cscs/env-clang-13.sh

@@ -30,6 +30,7 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS=ON"
 configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON"
 configure_extra_options+=" -DHPX_WITH_SPINLOCK_DEADLOCK_DETECTION=ON"
 configure_extra_options+=" -DHPX_WITH_UNITY_BUILD=ON"
+configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON"
 
 # enable extra counters to verify everything compiles
 configure_extra_options+=" -DHPX_WITH_BACKGROUND_THREAD_COUNTERS=ON"

.jenkins/lsu/batch.sh

@@ -22,6 +22,7 @@ ulimit -l unlimited
 set +e
 ctest \
     ${ctest_extra_args} \
+    --output-on-failure \
     -S ${src_dir}/.jenkins/lsu/ctest.cmake \
     -DCTEST_CONFIGURE_EXTRA_OPTIONS="${configure_extra_options}" \
     -DCTEST_BUILD_CONFIGURATION_NAME="${configuration_name_with_build_type}" \

.jenkins/lsu/env-clang-11.sh

@@ -31,3 +31,5 @@ configure_extra_options+=" -DHPX_WITH_LOGGING=OFF"
 # Make sure HWLOC does not report 'cores'. This is purely an option to enable
 # testing the topology code under conditions close to those on FreeBSD.
 configure_extra_options+=" -DHPX_TOPOLOGY_WITH_ADDITIONAL_HWLOC_TESTING=ON"
+
+configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON"

.jenkins/lsu/env-clang-12.sh

@@ -32,3 +32,5 @@ configure_extra_options+=" -DHPX_WITH_LOGGING=OFF"
 # Make sure HWLOC does not report 'cores'. This is purely an option to enable
 # testing the topology code under conditions close to those on FreeBSD.
 configure_extra_options+=" -DHPX_TOPOLOGY_WITH_ADDITIONAL_HWLOC_TESTING=ON"
+
+configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON"

.jenkins/lsu/env-gcc-10.sh

@@ -27,3 +27,5 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
 
 # The pwrapi library still needs to be set up properly on rostam
 # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON"
+
+configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON"

.jenkins/lsu/env-gcc-9.sh

@@ -26,3 +26,5 @@ configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON"
 
 # The pwrapi library still needs to be set up properly on rostam
 # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON"
+
+configure_extra_options+=" -DHPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD=ON"

components/iostreams/src/server/output_stream.cpp

@@ -13,6 +13,7 @@
 #include <hpx/serialization/serialize.hpp>
 #include <hpx/serialization/shared_ptr.hpp>
 #include <hpx/serialization/vector.hpp>
+#include <hpx/threading_base/thread_data.hpp>
 #include <hpx/threading_base/thread_helpers.hpp>
 
 #include <hpx/components/iostreams/server/buffer.hpp>
@@ -25,11 +26,12 @@
 #include <memory>
 #include <utility>
 
-namespace hpx { namespace iostreams { namespace detail {
+namespace hpx::iostreams::detail {
+
     void buffer::save(serialization::output_archive& ar, unsigned) const
     {
-        bool valid = (data_.get() && !data_->empty());
-        ar& valid;
+        bool const valid = (data_.get() && !data_->empty());
+        ar << valid;
         if (valid)
         {
             ar& data_;
@@ -39,15 +41,15 @@ namespace hpx { namespace iostreams { namespace detail {
     void buffer::load(serialization::input_archive& ar, unsigned)
     {
         bool valid = false;
-        ar& valid;
+        ar >> valid;
         if (valid)
         {
             ar& data_;
         }
     }
-}}}    // namespace hpx::iostreams::detail
+}    // namespace hpx::iostreams::detail
 
-namespace hpx { namespace iostreams { namespace server {
+namespace hpx::iostreams::server {
     ///////////////////////////////////////////////////////////////////////////
     void output_stream::call_write_async(std::uint32_t locality_id,
         std::uint64_t count, detail::buffer const& in, hpx::id_type /*this_id*/)
@@ -87,12 +89,13 @@ namespace hpx { namespace iostreams { namespace server {
     {    // {{{
         // Perform the IO in another OS thread.
         detail::buffer in(buf_in);
-        hpx::get_thread_pool("io_pool")->get_io_service().post(hpx::bind_front(
-            &output_stream::call_write_sync, this, locality_id, count,
-            std::ref(in), threads::thread_id_ref_type(threads::get_self_id())));
+        hpx::get_thread_pool("io_pool")->get_io_service().post(
+            hpx::bind_front(&output_stream::call_write_sync, this, locality_id,
+                count, std::ref(in),
+                threads::thread_id_ref_type(threads::get_outer_self_id())));
 
         // Sleep until the worker thread wakes us up.
         this_thread::suspend(threads::thread_schedule_state::suspended,
             "output_stream::write_sync");
     }    // }}}
-}}}      // namespace hpx::iostreams::server
+}    // namespace hpx::iostreams::server

docs/sphinx/manual/hpx_runtime_and_resources.rst

@@ -42,12 +42,11 @@ turned on, work stealing is done from queues associated with the same NUMA domai
 first, only after that work is stolen from other NUMA domains.
 
 This scheduler is enabled at build time by default using the FIFO
-(first-in-first-out) queing policy. This policy can be invoked using
+(first-in-first-out) queueing policy. This policy can be invoked using
 :option:`--hpx:queuing`\ ``local-priority-fifo``. The scheduler can also be
 enabled using the LIFO (last-in-first-out) policy. This is not the default
 policy and must be invoked using the command line option
-:option:`--hpx:queuing`\
-``=local-priority-lifo``.
+:option:`--hpx:queuing`\ ``local-priority-lifo``.
 
 Static priority scheduling policy
 ---------------------------------
@@ -63,7 +62,7 @@ robin fashion. There is no thread stealing in this policy.
 Local scheduling policy
 -----------------------
 
-* invoke using: :option:`--hpx:queuing`\ ``=local`` (or ``-ql``)
+* invoke using: :option:`--hpx:queuing`\ ``local`` (or ``-ql``)
 * flag to turn on for build: ``HPX_THREAD_SCHEDULERS=all`` or
   ``HPX_THREAD_SCHEDULERS=local``
 
@@ -73,7 +72,7 @@ thread pulls its tasks (user threads).
 Static scheduling policy
 ------------------------
 
-* invoke using: :option:`--hpx:queuing`\ ``=static``
+* invoke using: :option:`--hpx:queuing`\ ``static``
 * flag to turn on for build: ``HPX_THREAD_SCHEDULERS=all`` or
   ``HPX_THREAD_SCHEDULERS=static``
 
@@ -84,7 +83,7 @@ robin fashion. There is no thread stealing in this policy.
 Priority ABP scheduling policy
 ------------------------------
 
-* invoke using: :option:`--hpx:queuing`\ ``=abp-priority-fifo``
+* invoke using: :option:`--hpx:queuing`\ ``abp-priority-fifo``
 * flag to turn on for build: ``HPX_THREAD_SCHEDULERS=all`` or
   ``HPX_THREAD_SCHEDULERS=abp-priority``
 
@@ -102,8 +101,8 @@ domain first, only after that work is stolen from other NUMA domains.
 
 This scheduler can be used with two underlying queuing policies (FIFO:
 first-in-first-out, and LIFO: last-in-first-out). In order to use the LIFO
-policy use the command line option :option:`--hpx:queuing`\
-``=abp-priority-lifo``.
+policy use the command line option
+:option:`--hpx:queuing`\ ``=abp-priority-lifo``.
 
 ..
     Questions, concerns and notes:

docs/sphinx/manual/launching_and_configuring_hpx_applications.rst

@@ -366,6 +366,7 @@ The ``hpx.thread_queue`` configuration section
    min_add_new_count = ${HPX_THREAD_QUEUE_MIN_ADD_NEW_COUNT:10}
    max_add_new_count = ${HPX_THREAD_QUEUE_MAX_ADD_NEW_COUNT:10}
    max_delete_count = ${HPX_THREAD_QUEUE_MAX_DELETE_COUNT:1000}
+   max_recycled_stacks = ${HPX_THREAD_QUEUE_MAX_RECYCLED_STACKS:100}
 
 .. _ini_hpx_thread_queue:
 
@@ -392,6 +393,10 @@ The ``hpx.thread_queue`` configuration section
    * * ``hpx.thread_queue.max_delete_count``
      * The value of this property defines the number of terminated |hpx|
        threads to discard during each invocation of the corresponding function.
+   * * ``hpx.thread_queue.max_recucled_stacks``
+     * The value of this property defines the number of recycled threads that
+       can be accumulated before the stack segments of further recycled threads
+       are being deleted.
 
 The ``hpx.components`` configuration section
 ............................................

libs/core/async_base/include/hpx/async_base/scheduling_properties.hpp

@@ -64,6 +64,11 @@ namespace hpx::execution::experimental {
     {
     } with_priority{};
 
+    template <>
+    struct is_scheduling_property<with_priority_t> : std::true_type
+    {
+    };
+
     inline constexpr struct get_priority_t final
       : hpx::functional::detail::tag_fallback<get_priority_t>
     {
@@ -88,6 +93,11 @@ namespace hpx::execution::experimental {
     {
     } with_stacksize{};
 
+    template <>
+    struct is_scheduling_property<with_stacksize_t> : std::true_type
+    {
+    };
+
     inline constexpr struct get_stacksize_t final
       : hpx::functional::detail::tag_fallback<get_stacksize_t>
     {
@@ -112,6 +122,11 @@ namespace hpx::execution::experimental {
     {
     } with_hint{};
 
+    template <>
+    struct is_scheduling_property<with_hint_t> : std::true_type
+    {
+    };
+
     inline constexpr struct get_hint_t final
       : hpx::functional::detail::tag_fallback<get_hint_t>
     {
@@ -136,6 +151,11 @@ namespace hpx::execution::experimental {
     {
     } with_annotation{};
 
+    template <>
+    struct is_scheduling_property<with_annotation_t> : std::true_type
+    {
+    };
+
     inline constexpr struct get_annotation_t final
       : hpx::functional::detail::tag_fallback<get_annotation_t>
     {

libs/core/config/cmake/templates/config_version.hpp.in

@@ -1,4 +1,4 @@
-//  Copyright (c) 2007-2020 Hartmut Kaiser
+//  Copyright (c) 2007-2023 Hartmut Kaiser
 //  Copyright (c)      2011 Bryce Lelbach
 //  Copyright (c)      2013 Adrian Serio
 //
@@ -30,8 +30,9 @@
 /// ``HPX_VERSION_FULL & 0x00FF00 == HPX_VERSION_MINOR``
 /// ``HPX_VERSION_FULL & 0x0000FF == HPX_VERSION_SUBMINOR``
 #define HPX_VERSION_FULL                                                       \
-    ((HPX_VERSION_MAJOR << 16) | (HPX_VERSION_MINOR << 8) |                    \
-        HPX_VERSION_SUBMINOR)
+    ((HPX_PP_CAT(0x, HPX_VERSION_MAJOR) << 16) |                               \
+        (HPX_PP_CAT(0x, HPX_VERSION_MINOR) << 8) |                             \
+        HPX_PP_CAT(0x, HPX_VERSION_SUBMINOR))
 
 /// Evaluates to the release date of this HPX version in the format YYYYMMDD.
 #define HPX_VERSION_DATE @HPX_VERSION_DATE@

libs/core/config/include/hpx/config/threads_stack.hpp

@@ -12,6 +12,8 @@
 #include <hpx/config/debug.hpp>
 #include <hpx/config/defines.hpp>
 
+#include <cstddef>
+
 ///////////////////////////////////////////////////////////////////////////////
 
 // clang-format off
@@ -64,4 +66,8 @@
 #if !defined(HPX_HUGE_STACK_SIZE)
 #  define HPX_HUGE_STACK_SIZE     0x2000000       // 32MByte
 #endif
+
+#if !defined(HPX_THREAD_QUEUE_MAX_RECYCLED_STACKS)
+#  define HPX_THREAD_QUEUE_MAX_RECYCLED_STACKS std::size_t(100)
+#endif
 // clang-format on

libs/core/coroutines/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2020 The STE||AR-Group
+# Copyright (c) 2019-2023 The STE||AR-Group
 #
 # SPDX-License-Identifier: BSL-1.0
 # Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -14,6 +14,23 @@ hpx_option(
   MODULE COROUTINES
 )
 
+hpx_option(
+  HPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD
+  BOOL
+  "Futures attempt to run associated threads directly if those have not been started (default: OFF)"
+  OFF
+  CATEGORY "Thread Manager"
+  ADVANCED
+  MODULE COROUTINES
+)
+
+if(HPX_COROUTINES_WITH_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD)
+  hpx_add_config_define_namespace(
+    DEFINE HPX_COROUTINES_HAVE_THREAD_SCHEDULE_HINT_RUNS_AS_CHILD
+    NAMESPACE COROUTINES
+  )
+endif()
+
 set(coroutines_headers
     hpx/coroutines/coroutine.hpp
     hpx/coroutines/coroutine_fwd.hpp
@@ -29,6 +46,7 @@ set(coroutines_headers
     hpx/coroutines/detail/coroutine_impl.hpp
     hpx/coroutines/detail/coroutine_self.hpp
     hpx/coroutines/detail/coroutine_stackful_self.hpp
+    hpx/coroutines/detail/coroutine_stackful_self_direct.hpp
     hpx/coroutines/detail/coroutine_stackless_self.hpp
     hpx/coroutines/detail/get_stack_pointer.hpp
     hpx/coroutines/detail/posix_utility.hpp

libs/core/coroutines/include/hpx/coroutines/coroutine.hpp

@@ -1,5 +1,5 @@
 //  Copyright (c) 2006, Giovanni P. Deretta
-//  Copyright (c) 2007-2021 Hartmut Kaiser
+//  Copyright (c) 2007-2023 Hartmut Kaiser
 //
 //  This code may be used under either of the following two licences:
 //
@@ -140,6 +140,11 @@ namespace hpx::threads::coroutines {
             impl_.rebind(HPX_MOVE(f), HPX_MOVE(id));
         }
 
+        void delete_stack() noexcept
+        {
+            impl_.delete_stack();
+        }
+
         HPX_FORCEINLINE result_type operator()(arg_type arg = arg_type())
         {
             HPX_ASSERT(impl_.is_ready());
@@ -151,6 +156,12 @@ namespace hpx::threads::coroutines {
             return impl_.result();
         }
 
+        HPX_FORCEINLINE result_type invoke_directly(arg_type arg = arg_type())
+        {
+            HPX_ASSERT(impl_.is_ready());
+            return impl_.invoke_directly(arg);
+        }
+
         bool is_ready() const noexcept
         {
             return impl_.is_ready();

libs/core/coroutines/include/hpx/coroutines/detail/context_generic_context.hpp

@@ -227,10 +227,7 @@ namespace hpx::threads::coroutines {
 
             ~fcontext_context_impl()
             {
-                if (ctx_ && stack_pointer_)
-                {
-                    alloc_.deallocate(stack_pointer_, stack_size_);
-                }
+                delete_stack();
             }
 
             // Return the size of the reserved stack address space.
@@ -252,8 +249,11 @@ namespace hpx::threads::coroutines {
                 return (std::numeric_limits<std::ptrdiff_t>::max)();
             }
 #endif
-            void reset_stack() const
+            void reset_stack(bool direct_execution)
             {
+                if (direct_execution)
+                    return;
+
                 if (ctx_)
                 {
 #if defined(HPX_USE_POSIX_STACK_UTILITIES)
@@ -271,9 +271,19 @@ namespace hpx::threads::coroutines {
                 }
             }
 
+            void delete_stack() noexcept
+            {
+                if (stack_pointer_)
+                {
+                    alloc_.deallocate(stack_pointer_, stack_size_);
+                    stack_pointer_ = nullptr;
+                    ctx_ = nullptr;
+                }
+            }
+
             void rebind_stack()
             {
-                if (ctx_)
+                if (stack_pointer_)
                 {
 #if defined(HPX_HAVE_COROUTINE_COUNTERS)
                     increment_stack_recycle_count();