Reduce MAX_TERMINATED_THREADS default, improve memory use on manycore…

CMakeLists.txt

@@ -543,6 +543,17 @@ if(HPX_WITH_SCHEDULER_LOCAL_STORAGE)
   hpx_add_config_define(HPX_HAVE_SCHEDULER_LOCAL_STORAGE)
 endif()
 
+# Count number of terminated threads before forcefully cleaning up all of
+# them. Note: terminated threads are cleaned up either when this number is
+# reached for a particular thread queue or if the HPX_BUSY_LOOP_COUNT_MAX is
+# reached, which will clean up the terminated threads for _all_ thread queues.
+hpx_option(HPX_SCHEDULER_MAX_TERMINATED_THREADS STRING
+  "Maximum number of terminated threads collected before those are cleaned up (default: 100)"
+  "100" CATEGORY "Thread Manager" ADVANCED)
+
+hpx_add_config_define(HPX_SCHEDULER_MAX_TERMINATED_THREADS
+  ${HPX_SCHEDULER_MAX_TERMINATED_THREADS})
+
 hpx_option(HPX_WITH_SWAP_CONTEXT_EMULATION BOOL
   "Emulate SwapContext API for coroutines (default: OFF)"
   OFF CATEGORY "Thread Manager" ADVANCED)

hpx/config.hpp

@@ -1,4 +1,4 @@
-//  Copyright (c) 2007-2014 Hartmut Kaiser
+//  Copyright (c) 2007-2017 Hartmut Kaiser
 //  Copyright (c) 2011 Bryce Lelbach
 //
 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -385,15 +385,6 @@
 #  define HPX_BUSY_LOOP_COUNT_MAX 2000
 #endif
 
-///////////////////////////////////////////////////////////////////////////////
-// Count number of terminated threads before forcefully cleaning up all of
-// them. Note: terminated threads are cleaned up either when this number is
-// reached for a particular thread queue or if the HPX_BUSY_LOOP_COUNT_MAX is
-// reached, which will clean up the terminated threads for _all_ thread queues.
-#if !defined(HPX_MAX_TERMINATED_THREADS)
-#  define HPX_MAX_TERMINATED_THREADS 1000
-#endif
-
 ///////////////////////////////////////////////////////////////////////////////
 #if !defined(HPX_WRAPPER_HEAP_STEP)
 #  define HPX_WRAPPER_HEAP_STEP 0xFFFFU

hpx/runtime/threads/policies/thread_queue.hpp

@@ -1,4 +1,4 @@
-//  Copyright (c) 2007-2016 Hartmut Kaiser
+//  Copyright (c) 2007-2017 Hartmut Kaiser
 //  Copyright (c) 2011      Bryce Lelbach
 //
 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -37,6 +37,7 @@
 #include <map>
 #include <memory>
 #include <mutex>
+#include <string>
 #include <unordered_set>
 #include <utility>
 #include <vector>
@@ -117,6 +118,15 @@ namespace hpx { namespace threads { namespace policies
                     "hpx.thread_queue.max_delete_count", "1000"));
             return max_delete_count;
         }
+
+        inline int get_max_terminated_threads()
+        {
+            static int max_terminated_threads =
+                boost::lexical_cast<int>(hpx::get_config_entry(
+                    "hpx.thread_queue.max_terminated_threads",
+                    std::to_string(HPX_SCHEDULER_MAX_TERMINATED_THREADS)));
+            return max_terminated_threads;
+        }
     }
 
     ///////////////////////////////////////////////////////////////////////////
@@ -174,6 +184,9 @@ namespace hpx { namespace threads { namespace policies
         // number of terminated threads to discard
         int const max_delete_count;
 
+        // number of terminated threads to collect before cleaning them up
+        int const max_terminated_threads;
+
         // this is the type of a map holding all threads (except depleted ones)
         typedef std::unordered_set<thread_id_type> thread_map_type;
 
@@ -540,6 +553,7 @@ namespace hpx { namespace threads { namespace policies
             min_add_new_count(detail::get_min_add_new_count()),
             max_add_new_count(detail::get_max_add_new_count()),
             max_delete_count(detail::get_max_delete_count()),
+            max_terminated_threads(detail::get_max_terminated_threads()),
             thread_map_count_(0),
             work_items_(128, queue_num),
             work_items_count_(0),
@@ -882,7 +896,7 @@ namespace hpx { namespace threads { namespace policies
                 terminated_items_.push(thrd);
 
                 std::int64_t count = ++terminated_items_count_;
-                if (count > HPX_MAX_TERMINATED_THREADS)
+                if (count > max_terminated_threads)
                 {
                     cleanup_terminated(true);   // clean up all terminated threads
                 }

src/util/runtime_configuration.cpp

@@ -1,4 +1,4 @@
-//  Copyright (c) 2005-2016 Hartmut Kaiser
+//  Copyright (c) 2005-2017 Hartmut Kaiser
 //  Copyright (c)      2011 Bryce Adelstein-Lelbach
 //
 //  Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -239,6 +239,8 @@ namespace hpx { namespace util
             "min_add_new_count = ${HPX_THREAD_QUEUE_MIN_ADD_NEW_COUNT:10}",
             "max_add_new_count = ${HPX_THREAD_QUEUE_MAX_ADD_NEW_COUNT:10}",
             "max_delete_count = ${HPX_THREAD_QUEUE_MAX_DELETE_COUNT:1000}",
+            "max_terminated_threads = ${HPX_THREAD_QUEUE_MAX_TERMINATED_THREADS:"
+              HPX_PP_STRINGIZE(HPX_PP_EXPAND(HPX_SCHEDULER_MAX_TERMINATED_THREADS)) "}",
 
             "[hpx.commandline]",
             // enable aliasing