monitor threads entering or leaving the TBB global thread pool

cms-sw · Nov 23, 2017 · 4cfbe07 · 4cfbe07
1 parent d2d4d80
commit 4cfbe07
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 1 deletion.
diff --git a/HLTrigger/Timer/interface/FastTimerService.h b/HLTrigger/Timer/interface/FastTimerService.h
@@ -16,6 +16,7 @@
 // tbb headers
 #include <tbb/concurrent_unordered_set.h>
 #include <tbb/enumerable_thread_specific.h>
+#include <tbb/task_scheduler_observer.h>
 
 // CMSSW headers
 #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h"
@@ -79,7 +80,8 @@ Performance of std::chrono::high_resolution_clock
 */
 
 
-class FastTimerService {
+class FastTimerService : public tbb::task_scheduler_observer
+{
 public:
   FastTimerService(const edm::ParameterSet &, edm::ActivityRegistry & );
   ~FastTimerService() override;
@@ -214,6 +216,10 @@ class FastTimerService {
   void preEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
   void postEventReadFromSource(edm::StreamContext const&, edm::ModuleCallingContext const&);
 
+  // inherited from TBB task_scheduler_observer
+  void on_scheduler_entry(bool worker) final;
+  void on_scheduler_exit(bool worker) final;
+
 public:
   static void fillDescriptions(edm::ConfigurationDescriptions & descriptions);
 

diff --git a/HLTrigger/Timer/src/FastTimerService.cc b/HLTrigger/Timer/src/FastTimerService.cc
@@ -937,6 +937,10 @@ FastTimerService::FastTimerService(const edm::ParameterSet & config, edm::Activi
   highlight_module_psets_(      config.getUntrackedParameter<std::vector<edm::ParameterSet>>("highlightModules") ),
   highlight_modules_(           highlight_module_psets_.size())         // filled in postBeginJob()
 {
+  // start observing when a thread enters or leaves the TBB global thread arena
+  tbb::task_scheduler_observer::observe();
+
+  // register EDM call backs
   registry.watchPreallocate(                this, & FastTimerService::preallocate );
   registry.watchPreBeginJob(                this, & FastTimerService::preBeginJob );
   registry.watchPostBeginJob(               this, & FastTimerService::postBeginJob );
@@ -1872,6 +1876,22 @@ FastTimerService::postModuleStreamEndLumi(edm::StreamContext const&, edm::Module
   ignoredSignal(__func__);
 }
 
+void
+FastTimerService::on_scheduler_entry(bool worker)
+{
+  // initialise the measurement point for a thread that has newly joining the TBB pool
+  // FIXME any resources used or freed will be accounted to the next stream the uses this thread
+  thread().measure();
+}
+
+void
+FastTimerService::on_scheduler_exit(bool worker)
+{
+  // account any resources used or freed by the thread before leaving the TBB pool.
+  // FIXME arbitrarility use the first stream because there is no global measurement
+  auto & stream = streams_.front();
+  thread().measure_and_accumulate(stream.overhead);
+}
 
 FastTimerService::Measurement &
 FastTimerService::thread()