Merge pull request #3152 from msimberg/suspend-documentation

Documentation for runtime suspension

docs/CMakeLists.txt

@@ -47,6 +47,7 @@ set(doxygen_dependencies
     "${PROJECT_SOURCE_DIR}/hpx/hpx_init.hpp"
     "${PROJECT_SOURCE_DIR}/hpx/hpx_start.hpp"
     "${PROJECT_SOURCE_DIR}/hpx/hpx_finalize.hpp"
+    "${PROJECT_SOURCE_DIR}/hpx/hpx_suspend.hpp"
     "${PROJECT_SOURCE_DIR}/hpx/error.hpp"
     "${PROJECT_SOURCE_DIR}/hpx/error_code.hpp"
     "${PROJECT_SOURCE_DIR}/hpx/exception.hpp"
@@ -172,6 +173,7 @@ set(doxygen_dependencies
     "${PROJECT_SOURCE_DIR}/hpx/runtime/threads/thread_enums.hpp"
     "${PROJECT_SOURCE_DIR}/hpx/runtime/threads/thread_data_fwd.hpp"
     "${PROJECT_SOURCE_DIR}/hpx/runtime/threads/thread_helpers.hpp"
+    "${PROJECT_SOURCE_DIR}/hpx/runtime/threads/thread_pool_base.hpp"
     "${PROJECT_SOURCE_DIR}/hpx/lcos/barrier.hpp"
     "${PROJECT_SOURCE_DIR}/hpx/lcos/broadcast.hpp"
     "${PROJECT_SOURCE_DIR}/hpx/lcos/fold.hpp"

docs/hpx.idx

@@ -92,6 +92,9 @@ get_thread_priority_name              "" "hpx\.threads\.get_thread_priority_name
 get_thread_state_ex_name              "" "hpx\.threads\.get_thread_state_ex_name.*"
 get_stack_size_name                   "" "hpx\.threads\.get_stack_size_name.*"
 
+# hpx/runtime/threads/thread_pool_base.hpp
+thread_pool_base                      "" "hpx\.runtime\.threads\.thread_pool_base.*"
+
 # hpx/components/component_storage/migrate_from_storage.hpp
 migrate_from_storage                  "" "hpx\.components\.migrate_from_s.*"
 
@@ -588,6 +591,10 @@ init                                  "" "hpx\.init_id.*"
 # hpx/hpx_start.hpp
 start                                 "" "hpx\.start_id.*"
 
+# hpx/hpx_suspend.hpp
+resume                                "" "hpx\.suspend.*"
+suspend                               "" "hpx\.resume.*"
+
 # hpx/performance_counters/manage_counter_type.hpp
 install_counter_type                  "" "hpx\.performance_counters\.install_counter_type.*"
 

docs/manual/applications.qbk

@@ -187,4 +187,50 @@ available, such as for instance to provide your own entry point function instead
 of [funcref hpx_main `hpx_main`]. Please refer to the function documentation for
 more details (see: [headerref hpx/hpx_start.hpp `hpx/hpx_start.hpp`]).
 
+[heading:flexible Suspending and resuming the __hpx__ runtime]
+
+In some applications it is required to combine __hpx__ with other runtimes. To
+support this use case __hpx__ provides two functions: [funcref hpx::suspend
+`hpx::suspend`] and [funcref hpx::resume `hpx::resume`]. [funcref hpx::resume
+`hpx::suspend`] is a blocking call which will wait for all scheduled tasks to
+finish executing and then put the thread pool OS threads to sleep. [funcref
+hpx::resume `hpx::resume`] simply wakes up the sleeping threads so that they are
+ready to accept new work. [funcref hpx::suspend `hpx::suspend`] and [funcref
+hpx::resume `hpx::resume`] can be found in the header [headerref
+hpx/hpx_suspend.hpp `hpx/hpx_suspend.hpp`].
+
+    #include <``[headerref hpx/hpx_start.hpp `hpx/hpx_start.hpp`]``>
+    #include <``[headerref hpx/hpx_suspend.hpp `hpx/hpx_suspend.hpp`]``>
+
+    int main(int argc, char* argv[])
+    {
+        // Initialize HPX, don't run hpx_main
+        ``[funcref hpx::start `hpx::start`]``(nullptr, argc, argv);
+
+        // Schedule a function on the HPX runtime
+        ``[funcref hpx::async `hpx::async`]``(&my_function, ...);
+
+        // Wait for all tasks to finish, and suspend the HPX runtime
+        ``[funcref hpx::suspend `hpx::suspend`]``();
+
+        // Execute non-HPX code here
+
+        // Resume the HPX runtime
+        ``[funcref hpx::resume `hpx::resume`]``();
+
+        // Schedule more work on the HPX runtime
+
+        // Wait for hpx::finalize being called.
+        return ``[funcref hpx::stop `hpx::stop`]``();
+    }
+
+[note [funcref hpx::suspend `hpx::suspend`] does not wait for [funcref
+      hpx::finalize `hpx::finalize`] to be called. Only call [funcref
+      hpx::finalize `hpx::finalize`] when you wish to fully stop the __hpx__
+      runtime.]
+
+__hpx__ also supports suspending individual thread pools and threads. For
+details on how to do that see the documentation for [classref
+hpx::threads::thread_pool_base `thread_pool_base`].
+
 [endsect]

examples/resource_partitioner/oversubscribing_resource_partitioner.cpp

@@ -275,7 +275,7 @@ int main(int argc, char* argv[])
         [](hpx::threads::policies::callback_notifier& notifier,
             std::size_t num_threads, std::size_t thread_offset,
             std::size_t pool_index, std::string const& pool_name)
-        -> std::unique_ptr<hpx::threads::detail::thread_pool_base>
+        -> std::unique_ptr<hpx::threads::thread_pool_base>
         {
             std::cout << "User defined scheduler creation callback "
                       << std::endl;
@@ -287,7 +287,7 @@ int main(int argc, char* argv[])
             auto mode = scheduler_mode(scheduler_mode::do_background_work |
                 scheduler_mode::delay_exit);
 
-            std::unique_ptr<hpx::threads::detail::thread_pool_base> pool(
+            std::unique_ptr<hpx::threads::thread_pool_base> pool(
                 new hpx::threads::detail::scheduled_thread_pool<
                         high_priority_sched
                     >(std::move(scheduler), notifier,
@@ -310,7 +310,7 @@ int main(int argc, char* argv[])
             [](hpx::threads::policies::callback_notifier& notifier,
                 std::size_t num_threads, std::size_t thread_offset,
                 std::size_t pool_index, std::string const& pool_name)
-            -> std::unique_ptr<hpx::threads::detail::thread_pool_base>
+            -> std::unique_ptr<hpx::threads::thread_pool_base>
             {
                 std::cout << "User defined scheduler creation callback "
                           << std::endl;
@@ -320,7 +320,7 @@ int main(int argc, char* argv[])
 
                 auto mode = scheduler_mode(scheduler_mode::delay_exit);
 
-                std::unique_ptr<hpx::threads::detail::thread_pool_base> pool(
+                std::unique_ptr<hpx::threads::thread_pool_base> pool(
                     new hpx::threads::detail::scheduled_thread_pool<
                             high_priority_sched
                         >(std::move(scheduler), notifier,

examples/resource_partitioner/simple_resource_partitioner.cpp

@@ -277,7 +277,7 @@ int main(int argc, char* argv[])
         [](hpx::threads::policies::callback_notifier& notifier,
             std::size_t num_threads, std::size_t thread_offset,
             std::size_t pool_index, std::string const& pool_name)
-        -> std::unique_ptr<hpx::threads::detail::thread_pool_base>
+        -> std::unique_ptr<hpx::threads::thread_pool_base>
         {
             std::cout << "User defined scheduler creation callback "
                       << std::endl;
@@ -289,7 +289,7 @@ int main(int argc, char* argv[])
             auto mode = scheduler_mode(scheduler_mode::do_background_work |
                 scheduler_mode::delay_exit);
 
-            std::unique_ptr<hpx::threads::detail::thread_pool_base> pool(
+            std::unique_ptr<hpx::threads::thread_pool_base> pool(
                 new hpx::threads::detail::scheduled_thread_pool<
                         high_priority_sched
                     >(std::move(scheduler), notifier,
@@ -310,7 +310,7 @@ int main(int argc, char* argv[])
             [](hpx::threads::policies::callback_notifier& notifier,
                 std::size_t num_threads, std::size_t thread_offset,
                 std::size_t pool_index, std::string const& pool_name)
-            -> std::unique_ptr<hpx::threads::detail::thread_pool_base>
+            -> std::unique_ptr<hpx::threads::thread_pool_base>
             {
                 std::cout << "User defined scheduler creation callback "
                           << std::endl;
@@ -320,7 +320,7 @@ int main(int argc, char* argv[])
 
                 auto mode = scheduler_mode(scheduler_mode::delay_exit);
 
-                std::unique_ptr<hpx::threads::detail::thread_pool_base> pool(
+                std::unique_ptr<hpx::threads::thread_pool_base> pool(
                     new hpx::threads::detail::scheduled_thread_pool<
                             high_priority_sched
                         >(std::move(scheduler), notifier,

hpx/runtime/resource/partitioner_fwd.hpp

@@ -45,7 +45,7 @@ namespace hpx
 
         using scheduler_function =
             util::function_nonser<
-                std::unique_ptr<hpx::threads::detail::thread_pool_base>(
+                std::unique_ptr<hpx::threads::thread_pool_base>(
                     hpx::threads::policies::callback_notifier&,
                     std::size_t, std::size_t, std::size_t, std::string const&
                 )>;

hpx/runtime/thread_pool_helpers.hpp

@@ -13,10 +13,10 @@
 #include <cstddef>
 #include <string>
 
-namespace hpx { namespace threads { namespace detail
+namespace hpx { namespace threads
 {
     class HPX_EXPORT thread_pool_base;
-}}}
+}}
 
 namespace hpx { namespace resource
 {
@@ -44,12 +44,12 @@ namespace hpx { namespace resource
     HPX_API_EXPORT std::string const& get_pool_name(std::size_t pool_index);
 
     /// Return the name of the pool given its name
-    HPX_API_EXPORT threads::detail::thread_pool_base& get_thread_pool(
+    HPX_API_EXPORT threads::thread_pool_base& get_thread_pool(
         std::string const& pool_name);
 
     /// Return the thread pool given its internal index
-    HPX_API_EXPORT threads::detail::thread_pool_base& get_thread_pool(
+    HPX_API_EXPORT threads::thread_pool_base& get_thread_pool(
         std::size_t pool_index);
 }}
 
-#endif /*HPX_RUNTIME_GET_OS_THREAD_COUNT_HPP*/
+#endif

hpx/runtime/threads/detail/io_service_thread_pool.hpp

@@ -10,9 +10,9 @@
 #include <hpx/compat/barrier.hpp>
 #include <hpx/compat/mutex.hpp>
 #include <hpx/lcos/future.hpp>
-#include <hpx/runtime/threads/detail/thread_pool_base.hpp>
 #include <hpx/runtime/threads/policies/callback_notifier.hpp>
 #include <hpx/runtime/threads/policies/scheduler_mode.hpp>
+#include <hpx/runtime/threads/thread_pool_base.hpp>
 #include <hpx/state.hpp>
 #include <hpx/util/io_service_pool.hpp>
 

hpx/runtime/threads/detail/scheduled_thread_pool.hpp

@@ -12,9 +12,9 @@
 #include <hpx/compat/thread.hpp>
 #include <hpx/error_code.hpp>
 #include <hpx/lcos/future.hpp>
-#include <hpx/runtime/threads/detail/thread_pool_base.hpp>
 #include <hpx/runtime/threads/policies/callback_notifier.hpp>
 #include <hpx/runtime/threads/policies/scheduler_base.hpp>
+#include <hpx/runtime/threads/thread_pool_base.hpp>
 #include <hpx/util/assert.hpp>
 
 #include <atomic>
@@ -37,7 +37,7 @@ namespace hpx { namespace threads { namespace detail
 
     ///////////////////////////////////////////////////////////////////////////
     template <typename Scheduler>
-    class scheduled_thread_pool : public thread_pool_base
+    class scheduled_thread_pool : public hpx::threads::thread_pool_base
     {
     public:
         ///////////////////////////////////////////////////////////////////

hpx/runtime/threads/executors/pool_executor.hpp

@@ -7,8 +7,8 @@
 #define HPX_RUNTIME_THREADS_CUSTOMIZED_POOL_EXECUTOR
 
 #include <hpx/compat/mutex.hpp>
-#include <hpx/runtime/threads/detail/thread_pool_base.hpp>
 #include <hpx/runtime/threads/thread_executor.hpp>
+#include <hpx/runtime/threads/thread_pool_base.hpp>
 #include <hpx/util/steady_clock.hpp>
 #include <hpx/util/thread_description.hpp>
 #include <hpx/util/unique_function.hpp>
@@ -77,7 +77,7 @@ namespace hpx { namespace threads { namespace executors
                 closure_type func);
 
         private:
-            typedef hpx::threads::detail::thread_pool_base pool_type;
+            typedef hpx::threads::thread_pool_base pool_type;
 
             // the scheduler used by this executor
             pool_type& pool_;

hpx/runtime/threads/policies/scheduler_base.hpp

@@ -12,9 +12,9 @@
 #include <hpx/runtime/agas/interface.hpp>
 #include <hpx/runtime/parcelset_fwd.hpp>
 #include <hpx/runtime/resource/detail/partitioner.hpp>
-#include <hpx/runtime/threads/detail/thread_pool_base.hpp>
 #include <hpx/runtime/threads/policies/scheduler_mode.hpp>
 #include <hpx/runtime/threads/thread_init_data.hpp>
+#include <hpx/runtime/threads/thread_pool_base.hpp>
 #include <hpx/state.hpp>
 #include <hpx/util/assert.hpp>
 #include <hpx/util_fwd.hpp>
@@ -92,13 +92,13 @@ namespace hpx { namespace threads { namespace policies
         {
         }
 
-        threads::detail::thread_pool_base *get_parent_pool()
+        threads::thread_pool_base *get_parent_pool()
         {
             HPX_ASSERT(parent_pool_ != nullptr);
             return parent_pool_;
         }
 
-        void set_parent_pool(threads::detail::thread_pool_base *p)
+        void set_parent_pool(threads::thread_pool_base *p)
         {
             HPX_ASSERT(parent_pool_ == nullptr);
             parent_pool_ = p;
@@ -431,7 +431,7 @@ namespace hpx { namespace threads { namespace policies
         char const* description_;
 
         // the pool that owns this scheduler
-        threads::detail::thread_pool_base *parent_pool_;
+        threads::thread_pool_base *parent_pool_;
 
         std::atomic<std::int64_t> background_thread_count_;
 

hpx/runtime/threads/thread_helpers.hpp

@@ -454,7 +454,7 @@ namespace hpx { namespace threads
     ///         If this function is called while the thread-manager is not
     ///         running, it will throw an \a hpx#exception with an error code of
     ///         \a hpx#invalid_status.
-    HPX_EXPORT threads::detail::thread_pool_base*
+    HPX_EXPORT threads::thread_pool_base*
         get_pool(thread_id_type const& id, error_code& ec = throws);
 
     /// \cond NOINTERNAL
@@ -673,8 +673,7 @@ namespace hpx { namespace this_thread
     ///         If this function is called while the thread-manager is not
     ///         running, it will throw an \a hpx#exception with an error code of
     ///         \a hpx#invalid_status.
-    HPX_EXPORT threads::detail::thread_pool_base*
-        get_pool(error_code& ec = throws);
+    HPX_EXPORT threads::thread_pool_base* get_pool(error_code& ec = throws);
 
     /// \cond NOINTERNAL
     // returns the remaining available stack space