Fixed #1028: HPX-thread cumulative count performance counters report …

…incorrect value

docs/manual/existing_performance_counters.qbk

@@ -410,7 +410,7 @@ system and application performance.
           `locality#*` is defining the locality for which the
           overall number of retired __hpx__-threads should be queried for. The
           locality id (given by `*`) is a (zero based) number identifying the
-          locality
+          locality.
 
           `worker-thread#*` is defining the worker thread for which the
           overall number of retired __hpx__-threads should be queried for. The
@@ -427,6 +427,31 @@ system and application performance.
          is `worker-thread#*` the counter will return the overall number of
          retired __hpx__-threads for the specified worker thread.]
     ]
+    [   [`/threads/count/cumulative_phases`]
+        [`locality#*/total` or[br]
+         `locality#*/worker-thread#*`
+
+          where:[br]
+          `locality#*` is defining the locality for which the overall number
+          of executed __hpx__-thread phases (invocations) should be queried
+          for. The locality id (given by `*`) is a (zero based) number
+          identifying the locality.
+
+          `worker-thread#*` is defining the worker thread for which the
+          overall number of executed __hpx__-thread phases (invocations) should
+          be queried for. The worker thread number (given by the `*`) is a
+          (zero based) number identifying the worker thread. The number of
+          available worker threads is usually specified on the command line
+          for the application using the option [hpx_cmdline `--hpx:threads`].
+        ]
+        [None]
+        [Returns the overall number of executed (retired) __hpx__-threads on the
+         given locality since application start. If the instance name is `total`
+         the counter returns the accumulated number of retired __hpx__-threads
+         for all worker threads (cores) on that locality. If the instance name
+         is `worker-thread#*` the counter will return the overall number of
+         retired __hpx__-threads for the specified worker thread.]
+    ]
     [   [`/threads/count/instantaneous/<thread-state>`
 
           where:[br] `<thread-state>` is one of the following:

hpx/runtime/threads/detail/scheduling_loop.hpp

@@ -164,7 +164,8 @@ namespace hpx { namespace threads { namespace detail
     template <typename SchedulingPolicy>
     void scheduling_loop(std::size_t num_thread, SchedulingPolicy& scheduler,
         boost::atomic<hpx::state>& global_state, boost::int64_t& executed_threads,
-        boost::uint64_t& tfunc_time, boost::uint64_t& exec_time,
+        boost::int64_t& executed_thread_phases, boost::uint64_t& tfunc_time,
+        boost::uint64_t& exec_time,
         util::function_nonser<void()> const& cb = util::function_nonser<void()>())
     {
         util::itt::stack_context ctx;        // helper for itt support
@@ -221,6 +222,7 @@ namespace hpx { namespace threads { namespace detail
                                 exec_time += util::hardware::timestamp() - timestamp;
                             }
 
+                            ++executed_thread_phases;
                             tfunc_time = util::hardware::timestamp() - overall_timestamp;
                         }
                         else {
@@ -261,8 +263,9 @@ namespace hpx { namespace threads { namespace detail
 
                         // schedule this thread again, make sure it ends up at
                         // the end of the queue
-                        // REVIEW: Passing a specific target thread may screw
-                        // with the round robin queuing.
+                        //
+                        // REVIEW: Passing a specific target thread may set off
+                        // the round robin queuing.
                         scheduler.SchedulingPolicy::schedule_thread_last(thrd, num_thread);
                         scheduler.SchedulingPolicy::do_some_work(num_thread);
                     }
@@ -277,8 +280,9 @@ namespace hpx { namespace threads { namespace detail
                     // this might happen, if some thread has been added to the
                     // scheduler queue already but the state has not been reset
                     // yet
-                    // REVIEW: Passing a specific target thread may screw
-                    // with the round robin queuing.
+                    //
+                    // REVIEW: Passing a specific target thread may set off
+                    // the round robin queuing.
                     scheduler.SchedulingPolicy::schedule_thread(thrd, num_thread);
                 }
 

hpx/runtime/threads/threadmanager.hpp

@@ -412,6 +412,8 @@ namespace hpx { namespace threads
 
         virtual boost::int64_t get_executed_threads(
             std::size_t num = std::size_t(-1), bool reset = false) = 0;
+        virtual boost::int64_t get_executed_thread_phases(
+            std::size_t num = std::size_t(-1), bool reset = false) = 0;
 
 #if HPX_THREAD_MAINTAIN_THREAD_DATA
         /// The get_thread_data function is part of the thread related

hpx/runtime/threads/threadmanager_impl.hpp

@@ -459,8 +459,10 @@ namespace hpx { namespace threads
             scheduler_.on_error(num_thread, e);
         }
 
-        boost::int64_t get_executed_threads(std::size_t num = std::size_t(-1),
-            bool reset = false);
+        boost::int64_t get_executed_threads(
+            std::size_t num = std::size_t(-1), bool reset = false);
+        boost::int64_t get_executed_thread_phases(
+            std::size_t num = std::size_t(-1), bool reset = false);
 
     protected:
         ///
@@ -529,8 +531,9 @@ namespace hpx { namespace threads
         std::size_t num_threads_;
         boost::ptr_vector<boost::thread> threads_;
 
-        // count number of executed HPX-threads (invocations)
+        // count number of executed HPX-threads and thread phases (invocations)
         std::vector<boost::int64_t> executed_threads_;
+        std::vector<boost::int64_t> executed_thread_phases_;
         boost::atomic<long> thread_count_;
 
         boost::atomic<hpx::state> state_;   ///< thread manager state

src/runtime/threads/executors/thread_pool_executors.cpp

@@ -304,11 +304,11 @@ namespace hpx { namespace threads { namespace executors { namespace detail
 
             on_run_exit on_exit(current_concurrency_, shutdown_sem_);
 
-            boost::int64_t executed_threads = 0;
+            boost::int64_t executed_threads = 0, executed_thread_phases = 0;
             boost::uint64_t overall_times = 0, thread_times = 0;
             threads::detail::scheduling_loop(virt_core, scheduler_,
-                states_[virt_core], executed_threads, overall_times,
-                thread_times, &suspend_back_into_calling_context);
+                states_[virt_core], executed_threads, executed_thread_phases,
+                overall_times, thread_times, &suspend_back_into_calling_context);
 
 #if HPX_DEBUG != 0
             // the scheduling_loop is allowed to exit only if no more HPX

src/runtime/threads/threadmanager.cpp

@@ -1047,6 +1047,14 @@ namespace hpx { namespace threads
                   static_cast<std::size_t>(paths.instanceindex_), _1),
               "worker-thread", shepherd_count
             },
+            // /threads{locality#%d/total}/count/cumulative_phases
+            // /threads{locality#%d/worker-thread%d}/count/cumulative_phases
+            { "count/cumulative_phases",
+              HPX_STD_BIND(&ti::get_executed_thread_phases, this, -1, _1),
+              HPX_STD_BIND(&ti::get_executed_thread_phases, this,
+                  static_cast<std::size_t>(paths.instanceindex_), _1),
+              "worker-thread", shepherd_count
+            },
             // /threads{locality#%d/total}/count/instantaneous/all
             // /threads{locality#%d/worker-thread%d}/count/instantaneous/all
             { "count/instantaneous/all",
@@ -1289,8 +1297,8 @@ namespace hpx { namespace threads
 
         // run main scheduling loop until terminated
         detail::scheduling_loop(num_thread, scheduler_, state_,
-            executed_threads_[num_thread], tfunc_times[num_thread],
-            exec_times[num_thread]);
+            executed_threads_[num_thread], executed_thread_phases_[num_thread],
+            tfunc_times[num_thread], exec_times[num_thread]);
 
 #if HPX_DEBUG != 0
         // the OS thread is allowed to exit only if no more HPX threads exist
@@ -1321,6 +1329,7 @@ namespace hpx { namespace threads
         timer_pool_.run(false);
 
         executed_threads_.resize(num_threads);
+        executed_thread_phases_.resize(num_threads);
         tfunc_times.resize(num_threads);
         exec_times.resize(num_threads);
 
@@ -1445,6 +1454,27 @@ namespace hpx { namespace threads
         return result;
     }
 
+    template <typename SchedulingPolicy, typename NotificationPolicy>
+    boost::int64_t threadmanager_impl<SchedulingPolicy, NotificationPolicy>::
+        get_executed_thread_phases(std::size_t num, bool reset)
+    {
+        boost::int64_t result = 0;
+        if (num != std::size_t(-1)) {
+            result = executed_thread_phases_[num];
+            if (reset)
+                executed_thread_phases_[num] = 0;
+            return result;
+        }
+
+        result = std::accumulate(executed_thread_phases_.begin(),
+            executed_thread_phases_.end(), 0LL);
+        if (reset) {
+            std::fill(executed_thread_phases_.begin(),
+                executed_thread_phases_.end(), 0LL);
+        }
+        return result;
+    }
+
     ///////////////////////////////////////////////////////////////////////////
     template <typename SchedulingPolicy, typename NotificationPolicy>
     boost::int64_t threadmanager_impl<SchedulingPolicy, NotificationPolicy>::