Fixing 1288

docs/manual/existing_performance_counters.qbk

@@ -502,6 +502,34 @@ system and application performance.
          `HPX_THREAD_MAINTAIN_CUMULATIVE_COUNTS` is set to `ON`
          (default: ON).]
     ]
+    [   [`/threads/time/average`]
+        [`locality#*/total` or[br]
+         `locality#*/worker-thread#*`
+
+          where:[br]
+          `locality#*` is defining the locality for which the
+          average time spent executing one __hpx__-thread should be queried for.
+          The locality id (given by `*`) is a (zero based) number identifying the
+          locality.
+
+          `worker-thread#*` is defining the worker thread for which the
+          average time spent executing one __hpx__-thread should be queried for. The
+          worker thread number (given by the `*`) is a (zero based) number
+          identifying the worker thread. The number of available worker threads
+          is usually specified on the command line for the application using the
+          option [hpx_cmdline `--hpx:threads`].
+        ]
+        [None]
+        [Returns the average time spent executing one __hpx__-thread on the
+         given locality since application start. If the instance name is `total`
+         the counter returns the average time spent executing one __hpx__-thread
+         for all worker threads (cores) on that locality. If the instance name
+         is `worker-thread#*` the counter will return the average time spent
+         executing one __hpx__-thread for all worker threads separately.
+         This counter is available only if the configuration time constants
+         `HPX_THREAD_MAINTAIN_CUMULATIVE_COUNTS` (default: ON) and
+         `HPX_THREAD_MAINTAIN_IDLE_RATES` are set to `ON` (default: OFF).]
+    ]
     [   [`/threads/count/cumulative-phases`]
         [`locality#*/total` or[br]
          `locality#*/worker-thread#*`
@@ -530,6 +558,35 @@ system and application performance.
          This counter is available only if the configuration time constant
          `HPX_THREAD_MAINTAIN_CUMULATIVE_COUNTS` is set to `ON` (default: ON).]
     ]
+    [   [`/threads/time/average-phase`]
+        [`locality#*/total` or[br]
+         `locality#*/worker-thread#*`
+
+          where:[br]
+          `locality#*` is defining the locality for which the average time spent
+          executing one __hpx__-thread phase (invocation) should be queried
+          for. The locality id (given by `*`) is a (zero based) number
+          identifying the locality.
+
+          `worker-thread#*` is defining the worker thread for which the
+          average time executing one __hpx__-thread phase (invocation) should
+          be queried for. The worker thread number (given by the `*`) is a
+          (zero based) number identifying the worker thread. The number of
+          available worker threads is usually specified on the command line
+          for the application using the option [hpx_cmdline `--hpx:threads`].
+        ]
+        [None]
+        [Returns the average time spent executing one __hpx__-thread phase
+         (invocation) on the given locality since application start. If the
+         instance name is `total` the counter returns the average time spent
+         executing one __hpx__-thread phase (invocation) for all worker threads
+         (cores) on that locality. If the instance name is `worker-thread#*`
+         the counter will return the average time spent executing one
+         __hpx__-thread phase for all worker threads separately.
+         This counter is available only if the configuration time constants
+         `HPX_THREAD_MAINTAIN_CUMULATIVE_COUNTS` (default: ON) and
+         `HPX_THREAD_MAINTAIN_IDLE_RATES` are set to `ON` (default: OFF).]
+    ]
     [   [`/threads/count/instantaneous/<thread-state>`
 
           where:[br] `<thread-state>` is one of the following:

hpx/runtime/threads/threadmanager.hpp

@@ -426,6 +426,13 @@ namespace hpx { namespace threads
             std::size_t num = std::size_t(-1), bool reset = false) = 0;
         virtual boost::int64_t get_executed_thread_phases(
             std::size_t num = std::size_t(-1), bool reset = false) = 0;
+
+#ifdef HPX_THREAD_MAINTAIN_IDLE_RATES
+        virtual boost::int64_t get_thread_phase_duration(
+            std::size_t num = std::size_t(-1), bool reset = false) = 0;
+        virtual boost::int64_t get_thread_duration(
+            std::size_t num = std::size_t(-1), bool reset = false) = 0;
+#endif
 #endif
 
 #if defined(HPX_THREAD_MAINTAIN_LOCAL_STORAGE)

hpx/runtime/threads/threadmanager_impl.hpp

@@ -488,6 +488,13 @@ namespace hpx { namespace threads
             std::size_t num = std::size_t(-1), bool reset = false);
         boost::int64_t get_executed_thread_phases(
             std::size_t num = std::size_t(-1), bool reset = false);
+
+#ifdef HPX_THREAD_MAINTAIN_IDLE_RATES
+        boost::int64_t get_thread_phase_duration(
+            std::size_t num = std::size_t(-1), bool reset = false);
+        boost::int64_t get_thread_duration(
+            std::size_t num = std::size_t(-1), bool reset = false);
+#endif
 #endif
 
     protected:
@@ -564,6 +571,10 @@ namespace hpx { namespace threads
         std::vector<boost::int64_t> executed_thread_phases_;
         boost::atomic<long> thread_count_;
 
+#if defined(HPX_THREAD_MAINTAIN_CUMULATIVE_COUNTS) && defined(HPX_THREAD_MAINTAIN_IDLE_RATES)
+        double timestamp_scale_;            ///< scale timestamps to nanoseconds
+#endif
+
         boost::atomic<hpx::state> state_;   ///< thread manager state
         util::io_service_pool& timer_pool_; ///< used for timed set_state
 

src/runtime/threads/threadmanager.cpp

@@ -1,4 +1,4 @@
-//  Copyright (c) 2007-2013 Hartmut Kaiser
+//  Copyright (c) 2007-2014 Hartmut Kaiser
 //  Copyright (c)      2011 Bryce Lelbach, Katelyn Kufahl
 //  Copyright (c) 2008-2009 Chirag Dekate, Anshul Tandon
 //
@@ -138,6 +138,9 @@ namespace hpx { namespace threads
       : startup_(NULL),
         num_threads_(num_threads),
         thread_count_(0),
+#if defined(HPX_THREAD_MAINTAIN_CUMULATIVE_COUNTS) && defined(HPX_THREAD_MAINTAIN_IDLE_RATES)
+        timestamp_scale_(1.),
+#endif
         state_(starting),
         timer_pool_(timer_pool),
         thread_logger_("threadmanager_impl::register_thread"),
@@ -1121,6 +1124,24 @@ namespace hpx { namespace threads
                   static_cast<std::size_t>(paths.instanceindex_), _1),
               "worker-thread", shepherd_count
             },
+#ifdef HPX_THREAD_MAINTAIN_IDLE_RATES
+            // /threads{locality#%d/total}/time/average
+            // /threads{locality#%d/worker-thread%d}/time/average
+            { "time/average",
+              util::bind(&ti::get_thread_duration, this, -1, _1),
+              util::bind(&ti::get_thread_duration, this,
+                  static_cast<std::size_t>(paths.instanceindex_), _1),
+              "worker-thread", shepherd_count
+            },
+            // /threads{locality#%d/total}/time/average-phase
+            // /threads{locality#%d/worker-thread%d}/time/average-phase
+            { "time/average-phase",
+              util::bind(&ti::get_thread_phase_duration, this, -1, _1),
+              util::bind(&ti::get_thread_phase_duration, this,
+                  static_cast<std::size_t>(paths.instanceindex_), _1),
+              "worker-thread", shepherd_count
+            },
+#endif
 #endif
             // /threads{locality#%d/total}/count/instantaneous/all
             // /threads{locality#%d/worker-thread%d}/count/instantaneous/all
@@ -1352,6 +1373,20 @@ namespace hpx { namespace threads
               &performance_counters::locality_thread_counter_discoverer,
               ""
             },
+#ifdef HPX_THREAD_MAINTAIN_IDLE_RATES
+            { "/threads/time/average", performance_counters::counter_raw,
+              "returns the average time spent executing one HPX-thread",
+              HPX_PERFORMANCE_COUNTER_V1, counts_creator,
+              &performance_counters::locality_thread_counter_discoverer,
+              "ns"
+            },
+            { "/threads/time/average-phase", performance_counters::counter_raw,
+              "returns the average time spent executing one HPX-thread phase",
+              HPX_PERFORMANCE_COUNTER_V1, counts_creator,
+              &performance_counters::locality_thread_counter_discoverer,
+              "ns"
+            },
+#endif
 #endif
             { "/threads/count/instantaneous/all", performance_counters::counter_raw,
               "returns the overall current number of HPX-threads instantiated at the "
@@ -1504,6 +1539,28 @@ namespace hpx { namespace threads
                 "threadmanager_impl::run", "number of threads is zero");
         }
 
+#if defined(HPX_THREAD_MAINTAIN_CUMULATIVE_COUNTS) && defined(HPX_THREAD_MAINTAIN_IDLE_RATES)
+        // scale timestamps to nanoseconds
+        boost::uint64_t base_timestamp = util::hardware::timestamp();
+        boost::uint64_t base_time = util::high_resolution_clock::now();
+        boost::uint64_t curr_timestamp = util::hardware::timestamp();
+        boost::uint64_t curr_time = util::high_resolution_clock::now();
+
+        while ((curr_time - base_time) <= 100000)
+        {
+            curr_timestamp = util::hardware::timestamp();
+            curr_time = util::high_resolution_clock::now();
+        }
+
+        if (curr_timestamp - base_timestamp != 0)
+        {
+            timestamp_scale_ = double(curr_time - base_time) /
+                double(curr_timestamp - base_timestamp);
+        }
+
+        LTM_(info) << "run: timestamp_scale: " << timestamp_scale_; //-V128
+#endif
+
         mutex_type::scoped_lock lk(mtx_);
         if (!threads_.empty() || (state_.load() == running))
             return true;    // do nothing if already running
@@ -1659,6 +1716,65 @@ namespace hpx { namespace threads
         }
         return result;
     }
+
+#ifdef HPX_THREAD_MAINTAIN_IDLE_RATES
+    template <typename SchedulingPolicy, typename NotificationPolicy>
+    boost::int64_t threadmanager_impl<SchedulingPolicy, NotificationPolicy>::
+        get_thread_phase_duration(std::size_t num, bool reset)
+    {
+        if (num != std::size_t(-1)) {
+            double exec_total = static_cast<double>(exec_times[num]);
+            double num_phases = static_cast<double>(executed_thread_phases_[num]);
+
+            if (reset) {
+                executed_thread_phases_[num] = 0;
+                tfunc_times[num] = boost::uint64_t(-1);
+            }
+            return boost::uint64_t((exec_total * timestamp_scale_)/ num_phases);
+        }
+
+        double exec_total = std::accumulate(exec_times.begin(),
+            exec_times.end(), 0.);
+        double num_phases = std::accumulate(executed_thread_phases_.begin(),
+            executed_thread_phases_.end(), 0.);
+
+        if (reset) {
+            std::fill(executed_thread_phases_.begin(),
+                executed_thread_phases_.end(), 0LL);
+            std::fill(tfunc_times.begin(), tfunc_times.end(),
+                boost::uint64_t(-1));
+        }
+        return boost::uint64_t((exec_total * timestamp_scale_)/ num_phases);
+    }
+
+    template <typename SchedulingPolicy, typename NotificationPolicy>
+    boost::int64_t threadmanager_impl<SchedulingPolicy, NotificationPolicy>::
+        get_thread_duration(std::size_t num, bool reset)
+    {
+        if (num != std::size_t(-1)) {
+            double exec_total = static_cast<double>(exec_times[num]);
+            double num_threads = static_cast<double>(executed_threads_[num]);
+
+            if (reset) {
+                executed_threads_[num] = 0;
+                tfunc_times[num] = boost::uint64_t(-1);
+            }
+            return boost::uint64_t((exec_total * timestamp_scale_)/ num_threads);
+        }
+
+        double exec_total = std::accumulate(exec_times.begin(),
+            exec_times.end(), 0.);
+        double num_threads = std::accumulate(executed_threads_.begin(),
+            executed_threads_.end(), 0.);
+
+        if (reset) {
+            std::fill(executed_threads_.begin(), executed_threads_.end(), 0LL);
+            std::fill(tfunc_times.begin(), tfunc_times.end(),
+                boost::uint64_t(-1));
+        }
+        return boost::uint64_t((exec_total * timestamp_scale_) / num_threads);
+    }
+#endif
 #endif
 
 #ifdef HPX_THREAD_MAINTAIN_IDLE_RATES