From 7280fe58855b45238352e737957286fdec35a939 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Mon, 16 Aug 2021 10:45:57 -0500 Subject: [PATCH 1/6] Added ThreadHandoff class --- .../Application/interface/ThreadHandoff.h | 99 +++++++++++++++ SimG4Core/Application/src/ThreadHandoff.cc | 119 ++++++++++++++++++ .../test/test_catch2_ThreadHandoff.cc | 22 ++++ 3 files changed, 240 insertions(+) create mode 100644 SimG4Core/Application/interface/ThreadHandoff.h create mode 100644 SimG4Core/Application/src/ThreadHandoff.cc create mode 100644 SimG4Core/Application/test/test_catch2_ThreadHandoff.cc diff --git a/SimG4Core/Application/interface/ThreadHandoff.h b/SimG4Core/Application/interface/ThreadHandoff.h new file mode 100644 index 0000000000000..09cb5cd1c16dc --- /dev/null +++ b/SimG4Core/Application/interface/ThreadHandoff.h @@ -0,0 +1,99 @@ +#ifndef SimG4Core_Application_ThreadHandoff_h +#define SimG4Core_Application_ThreadHandoff_h +// -*- C++ -*- +// +// Package: SimG4Core/Application +// Class : ThreadHandoff +// +/**\class ThreadHandoff ThreadHandoff.h "SimG4Core/Application/interface/ThreadHandoff.h" + + Description: [one line class summary] + + Usage: + + +*/ +// +// Original Author: Christopher Jones +// Created: Mon, 16 Aug 2021 13:51:53 GMT +// + +// system include files +#include +#include //strerror_r +#include +#include +#include + +// user include files + +// forward declarations + +namespace omt { +class ThreadHandoff { +public: + explicit ThreadHandoff(); + ~ThreadHandoff(); + + ThreadHandoff(const ThreadHandoff&) = delete; // stop default + const ThreadHandoff& operator=(const ThreadHandoff&) = delete; // stop default + + template + void runAndWait(F&& iF) { + Functor f{std::move(iF)}; + + std::unique_lock lck(m_mutex); + m_loopReady = false; + m_toRun = &f; + + m_threadHandoff.notify_one(); + + m_threadHandoff.wait(lck, [this]() { + return m_loopReady; + }); + auto e = f.exception(); + if(e) { std::rethrow_exception(e); } + } + + void stopThread() { + runAndWait([this]() { m_stopThread =true;}); + } + +private: + class FunctorBase { + public: + virtual ~FunctorBase() {} + virtual void execute() = 0; + }; + template + class Functor : public FunctorBase { + public: + explicit Functor(F&& iF): m_f(std::move(iF)) {} + void execute() final { + try { + m_f(); + }catch(...) { + m_except = std::current_exception(); + } + } + std::exception_ptr exception() { return m_except;} + private: + F m_f; + std::exception_ptr m_except; + }; + + + static void* threadLoop(void* iArgs); + + // ---------- member data -------------------------------- + pthread_t m_thread; + std::mutex m_mutex; + std::condition_variable m_threadHandoff; + + FunctorBase* m_toRun{nullptr}; + bool m_loopReady{false}; + bool m_stopThread{false}; + +}; +} +#endif diff --git a/SimG4Core/Application/src/ThreadHandoff.cc b/SimG4Core/Application/src/ThreadHandoff.cc new file mode 100644 index 0000000000000..1963181dbb0ba --- /dev/null +++ b/SimG4Core/Application/src/ThreadHandoff.cc @@ -0,0 +1,119 @@ +// -*- C++ -*- +// +// Package: SimG4Core/Application +// Class : ThreadHandoff +// +// Implementation: +// [Notes on implementation] +// +// Original Author: Christopher Jones +// Created: Mon, 16 Aug 2021 14:37:29 GMT +// + +// system include files + +// user include files +#include "SimG4Core/Application/interface/ThreadHandoff.h" +#include "FWCore/Utilities/interface/Exception.h" + +// +// constants, enums and typedefs +// + +namespace { + std::string errorMessage(int erno) { + std::array buffer; + strerror_r(erno, &buffer[0], buffer.size()); + buffer.back() = '\0'; + return std::string(&buffer[0]); + } +} +// +// static data member definitions +// + +// +// constructors and destructor +// +using namespace omt; + +ThreadHandoff::ThreadHandoff() { + pthread_attr_t attr; + int erno; + if( 0 != (erno = pthread_attr_init(&attr)) ) { + throw cms::Exception("ThreadInitFailed") <<"Failed to initialize thread attributes (" + < lk(m_mutex); + + erno = pthread_create(&m_thread, &attr, + threadLoop, this); + if( 0 != erno) { + throw cms::Exception("ThreadCreateFailed")<<" failed to create a pthread (" + <(iArgs); + { + std::unique_lock lck(theThis->m_mutex); + theThis->m_loopReady = true; + } + theThis->m_threadHandoff.notify_one(); + + std::unique_lock lck(theThis->m_mutex); + do { + theThis->m_toRun=nullptr; + theThis->m_threadHandoff.wait(lck, [theThis]() { return nullptr != theThis->m_toRun;}); + theThis->m_toRun->execute(); + theThis->m_loopReady=true; + theThis->m_threadHandoff.notify_one(); + } while(not theThis->m_stopThread); + theThis->m_loopReady=true; + return nullptr; +} diff --git a/SimG4Core/Application/test/test_catch2_ThreadHandoff.cc b/SimG4Core/Application/test/test_catch2_ThreadHandoff.cc new file mode 100644 index 0000000000000..a291c610f6bd3 --- /dev/null +++ b/SimG4Core/Application/test/test_catch2_ThreadHandoff.cc @@ -0,0 +1,22 @@ +#include "SimG4Core/Application/interface/ThreadHandoff.h" +#include "FWCore/Utilities/interface/Exception.h" +#define CATCH_CONFIG_MAIN +#include "catch.hpp" + +using namespace omt; +TEST_CASE("Test omt::ThreadHandoff", "[ThreadHandoff]") { + SECTION("Do nothing") { + ThreadHandoff th; + } + SECTION("Simple") { + ThreadHandoff th; + bool value = false; + th.runAndWait([&value]() { value = true;}); + REQUIRE(value == true); + } + + SECTION("Exception") { + ThreadHandoff th; + REQUIRE_THROWS_AS(th.runAndWait([]() { throw cms::Exception("Test"); }), cms::Exception); + } +} From e51cf0d1af7792fdcb2c3aa326c1bfe251a3cd8d Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Mon, 16 Aug 2021 12:17:56 -0500 Subject: [PATCH 2/6] Run G4 worker on its own dedicated thread --- .../Application/plugins/OscarMTProducer.cc | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/SimG4Core/Application/plugins/OscarMTProducer.cc b/SimG4Core/Application/plugins/OscarMTProducer.cc index d03a59bfd909d..23bbe90a2a668 100644 --- a/SimG4Core/Application/plugins/OscarMTProducer.cc +++ b/SimG4Core/Application/plugins/OscarMTProducer.cc @@ -14,6 +14,7 @@ #include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/ServiceRegistry/interface/Service.h" +#include "FWCore/ServiceRegistry/interface/ServiceRegistry.h" #include "FWCore/Utilities/interface/RandomNumberGenerator.h" #include "FWCore/Utilities/interface/Exception.h" @@ -34,6 +35,8 @@ #include "SimDataFormats/TrackingHit/interface/PSimHitContainer.h" #include "SimDataFormats/CaloHit/interface/PCaloHitContainer.h" +#include "SimG4Core/Application/interface/ThreadHandoff.h" + #include "Randomize.hh" // for some reason void doesn't compile @@ -56,6 +59,7 @@ class OscarMTProducer : public edm::stream::EDProducer m_runManagerWorker; const OscarMTMasterThread* m_masterThread = nullptr; }; @@ -97,7 +101,11 @@ OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMaster // Random number generation not allowed here StaticRandomEngineSetUnset random(nullptr); - m_runManagerWorker = std::make_unique(p, consumesCollector()); + auto token = edm::ServiceRegistry::instance().presentToken(); + m_handoff.runAndWait([this, &p, token]() { + edm::ServiceRegistry::Operate guard{token}; + m_runManagerWorker = std::make_unique(p, consumesCollector()); + }); m_masterThread = ms; m_masterThread->callConsumes(consumesCollector()); @@ -165,7 +173,11 @@ OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMaster edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer is constructed"; } -OscarMTProducer::~OscarMTProducer() {} +OscarMTProducer::~OscarMTProducer() { + auto token = edm::ServiceRegistry::instance().presentToken(); + m_handoff.runAndWait([this, token]() { edm::ServiceRegistry::Operate guard{token}; + m_runManagerWorker.reset(); }); +} std::unique_ptr OscarMTProducer::initializeGlobalCache(const edm::ParameterSet& iConfig) { // Random number generation not allowed here @@ -198,8 +210,12 @@ void OscarMTProducer::globalEndJob(OscarMTMasterThread* masterThread) { void OscarMTProducer::beginRun(const edm::Run&, const edm::EventSetup& es) { edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::beginRun"; - m_runManagerWorker->beginRun(es); - m_runManagerWorker->initializeG4(m_masterThread->runManagerMasterPtr(), es); + auto token = edm::ServiceRegistry::instance().presentToken(); + m_handoff.runAndWait([this, &es, token]() { + edm::ServiceRegistry::Operate guard{token}; + m_runManagerWorker->beginRun(es); + m_runManagerWorker->initializeG4(m_masterThread->runManagerMasterPtr(), es); + }); edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::beginRun done"; } @@ -207,7 +223,10 @@ void OscarMTProducer::endRun(const edm::Run&, const edm::EventSetup&) { // Random number generation not allowed here StaticRandomEngineSetUnset random(nullptr); edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::endRun"; - m_runManagerWorker->endRun(); + auto token = edm::ServiceRegistry::instance().presentToken(); + m_handoff.runAndWait([this, token]() { + edm::ServiceRegistry::Operate guard{token}; + m_runManagerWorker->endRun();}); edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::endRun done"; } @@ -221,7 +240,11 @@ void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) { std::unique_ptr evt; try { - evt = m_runManagerWorker->produce(e, es, globalCache()->runManagerMaster()); + auto token = edm::ServiceRegistry::instance().presentToken(); + m_handoff.runAndWait([this, &e, &es, &evt, token]() { + edm::ServiceRegistry::Operate guard{token}; + evt = m_runManagerWorker->produce(e, es, globalCache()->runManagerMaster()); + }); } catch (const SimG4Exception& simg4ex) { edm::LogWarning("SimG4CoreApplication") << "SimG4Exception caght! " << simg4ex.what(); From aa646866ca53fb673f43647c13036d3619bb4bb9 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Mon, 16 Aug 2021 14:13:42 -0500 Subject: [PATCH 3/6] applied code format --- .../Application/interface/ThreadHandoff.h | 113 +++++++++--------- .../Application/plugins/OscarMTProducer.cc | 29 +++-- SimG4Core/Application/src/ThreadHandoff.cc | 64 +++++----- .../test/test_catch2_ThreadHandoff.cc | 6 +- 4 files changed, 105 insertions(+), 107 deletions(-) diff --git a/SimG4Core/Application/interface/ThreadHandoff.h b/SimG4Core/Application/interface/ThreadHandoff.h index 09cb5cd1c16dc..9b9e0b9209090 100644 --- a/SimG4Core/Application/interface/ThreadHandoff.h +++ b/SimG4Core/Application/interface/ThreadHandoff.h @@ -20,7 +20,7 @@ // system include files #include -#include //strerror_r +#include //strerror_r #include #include #include @@ -30,70 +30,69 @@ // forward declarations namespace omt { -class ThreadHandoff { -public: - explicit ThreadHandoff(); - ~ThreadHandoff(); + class ThreadHandoff { + public: + explicit ThreadHandoff(); + ~ThreadHandoff(); - ThreadHandoff(const ThreadHandoff&) = delete; // stop default - const ThreadHandoff& operator=(const ThreadHandoff&) = delete; // stop default + ThreadHandoff(const ThreadHandoff&) = delete; // stop default + const ThreadHandoff& operator=(const ThreadHandoff&) = delete; // stop default - template + template void runAndWait(F&& iF) { - Functor f{std::move(iF)}; - - std::unique_lock lck(m_mutex); - m_loopReady = false; - m_toRun = &f; - - m_threadHandoff.notify_one(); - - m_threadHandoff.wait(lck, [this]() { - return m_loopReady; - }); - auto e = f.exception(); - if(e) { std::rethrow_exception(e); } - } - - void stopThread() { - runAndWait([this]() { m_stopThread =true;}); - } - -private: - class FunctorBase { - public: - virtual ~FunctorBase() {} - virtual void execute() = 0; - }; - template - class Functor : public FunctorBase { - public: - explicit Functor(F&& iF): m_f(std::move(iF)) {} - void execute() final { - try { - m_f(); - }catch(...) { - m_except = std::current_exception(); + Functor f{std::move(iF)}; + + std::unique_lock lck(m_mutex); + m_loopReady = false; + m_toRun = &f; + + m_threadHandoff.notify_one(); + + m_threadHandoff.wait(lck, [this]() { return m_loopReady; }); + auto e = f.exception(); + if (e) { + std::rethrow_exception(e); } } - std::exception_ptr exception() { return m_except;} - private: - F m_f; - std::exception_ptr m_except; - }; + void stopThread() { + runAndWait([this]() { m_stopThread = true; }); + } - static void* threadLoop(void* iArgs); + private: + class FunctorBase { + public: + virtual ~FunctorBase() {} + virtual void execute() = 0; + }; + template + class Functor : public FunctorBase { + public: + explicit Functor(F&& iF) : m_f(std::move(iF)) {} + void execute() final { + try { + m_f(); + } catch (...) { + m_except = std::current_exception(); + } + } + std::exception_ptr exception() { return m_except; } - // ---------- member data -------------------------------- - pthread_t m_thread; - std::mutex m_mutex; - std::condition_variable m_threadHandoff; + private: + F m_f; + std::exception_ptr m_except; + }; - FunctorBase* m_toRun{nullptr}; - bool m_loopReady{false}; - bool m_stopThread{false}; + static void* threadLoop(void* iArgs); -}; -} + // ---------- member data -------------------------------- + pthread_t m_thread; + std::mutex m_mutex; + std::condition_variable m_threadHandoff; + + FunctorBase* m_toRun{nullptr}; + bool m_loopReady{false}; + bool m_stopThread{false}; + }; +} // namespace omt #endif diff --git a/SimG4Core/Application/plugins/OscarMTProducer.cc b/SimG4Core/Application/plugins/OscarMTProducer.cc index 23bbe90a2a668..54d3668e45200 100644 --- a/SimG4Core/Application/plugins/OscarMTProducer.cc +++ b/SimG4Core/Application/plugins/OscarMTProducer.cc @@ -103,9 +103,9 @@ OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMaster auto token = edm::ServiceRegistry::instance().presentToken(); m_handoff.runAndWait([this, &p, token]() { - edm::ServiceRegistry::Operate guard{token}; - m_runManagerWorker = std::make_unique(p, consumesCollector()); - }); + edm::ServiceRegistry::Operate guard{token}; + m_runManagerWorker = std::make_unique(p, consumesCollector()); + }); m_masterThread = ms; m_masterThread->callConsumes(consumesCollector()); @@ -175,8 +175,10 @@ OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMaster OscarMTProducer::~OscarMTProducer() { auto token = edm::ServiceRegistry::instance().presentToken(); - m_handoff.runAndWait([this, token]() { edm::ServiceRegistry::Operate guard{token}; - m_runManagerWorker.reset(); }); + m_handoff.runAndWait([this, token]() { + edm::ServiceRegistry::Operate guard{token}; + m_runManagerWorker.reset(); + }); } std::unique_ptr OscarMTProducer::initializeGlobalCache(const edm::ParameterSet& iConfig) { @@ -212,10 +214,10 @@ void OscarMTProducer::beginRun(const edm::Run&, const edm::EventSetup& es) { edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::beginRun"; auto token = edm::ServiceRegistry::instance().presentToken(); m_handoff.runAndWait([this, &es, token]() { - edm::ServiceRegistry::Operate guard{token}; - m_runManagerWorker->beginRun(es); - m_runManagerWorker->initializeG4(m_masterThread->runManagerMasterPtr(), es); - }); + edm::ServiceRegistry::Operate guard{token}; + m_runManagerWorker->beginRun(es); + m_runManagerWorker->initializeG4(m_masterThread->runManagerMasterPtr(), es); + }); edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::beginRun done"; } @@ -225,8 +227,9 @@ void OscarMTProducer::endRun(const edm::Run&, const edm::EventSetup&) { edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::endRun"; auto token = edm::ServiceRegistry::instance().presentToken(); m_handoff.runAndWait([this, token]() { - edm::ServiceRegistry::Operate guard{token}; - m_runManagerWorker->endRun();}); + edm::ServiceRegistry::Operate guard{token}; + m_runManagerWorker->endRun(); + }); edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::endRun done"; } @@ -243,8 +246,8 @@ void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) { auto token = edm::ServiceRegistry::instance().presentToken(); m_handoff.runAndWait([this, &e, &es, &evt, token]() { edm::ServiceRegistry::Operate guard{token}; - evt = m_runManagerWorker->produce(e, es, globalCache()->runManagerMaster()); - }); + evt = m_runManagerWorker->produce(e, es, globalCache()->runManagerMaster()); + }); } catch (const SimG4Exception& simg4ex) { edm::LogWarning("SimG4CoreApplication") << "SimG4Exception caght! " << simg4ex.what(); diff --git a/SimG4Core/Application/src/ThreadHandoff.cc b/SimG4Core/Application/src/ThreadHandoff.cc index 1963181dbb0ba..c4bab34e0354e 100644 --- a/SimG4Core/Application/src/ThreadHandoff.cc +++ b/SimG4Core/Application/src/ThreadHandoff.cc @@ -22,12 +22,12 @@ namespace { std::string errorMessage(int erno) { - std::array buffer; - strerror_r(erno, &buffer[0], buffer.size()); - buffer.back() = '\0'; - return std::string(&buffer[0]); + std::array buffer; + strerror_r(erno, &buffer[0], buffer.size()); + buffer.back() = '\0'; + return std::string(&buffer[0]); } -} +} // namespace // // static data member definitions // @@ -38,28 +38,26 @@ namespace { using namespace omt; ThreadHandoff::ThreadHandoff() { - pthread_attr_t attr; - int erno; - if( 0 != (erno = pthread_attr_init(&attr)) ) { - throw cms::Exception("ThreadInitFailed") <<"Failed to initialize thread attributes (" - < lk(m_mutex); + pthread_attr_t attr; + int erno; + if (0 != (erno = pthread_attr_init(&attr))) { + throw cms::Exception("ThreadInitFailed") + << "Failed to initialize thread attributes (" << erno << ") " << errorMessage(erno); + } + const int stackSize = 10 * 1024 * 1024; + + if (0 != (erno = pthread_attr_setstacksize(&attr, stackSize))) { + throw cms::Exception("ThreadStackSizeFailed") + << "Failed to set stack size " << stackSize << " " << errorMessage(erno); + } + std::unique_lock lk(m_mutex); - erno = pthread_create(&m_thread, &attr, - threadLoop, this); - if( 0 != erno) { - throw cms::Exception("ThreadCreateFailed")<<" failed to create a pthread (" - <m_loopReady = true; } theThis->m_threadHandoff.notify_one(); - + std::unique_lock lck(theThis->m_mutex); do { - theThis->m_toRun=nullptr; - theThis->m_threadHandoff.wait(lck, [theThis]() { return nullptr != theThis->m_toRun;}); + theThis->m_toRun = nullptr; + theThis->m_threadHandoff.wait(lck, [theThis]() { return nullptr != theThis->m_toRun; }); theThis->m_toRun->execute(); - theThis->m_loopReady=true; + theThis->m_loopReady = true; theThis->m_threadHandoff.notify_one(); - } while(not theThis->m_stopThread); - theThis->m_loopReady=true; + } while (not theThis->m_stopThread); + theThis->m_loopReady = true; return nullptr; } diff --git a/SimG4Core/Application/test/test_catch2_ThreadHandoff.cc b/SimG4Core/Application/test/test_catch2_ThreadHandoff.cc index a291c610f6bd3..5f930d811cba0 100644 --- a/SimG4Core/Application/test/test_catch2_ThreadHandoff.cc +++ b/SimG4Core/Application/test/test_catch2_ThreadHandoff.cc @@ -5,13 +5,11 @@ using namespace omt; TEST_CASE("Test omt::ThreadHandoff", "[ThreadHandoff]") { - SECTION("Do nothing") { - ThreadHandoff th; - } + SECTION("Do nothing") { ThreadHandoff th; } SECTION("Simple") { ThreadHandoff th; bool value = false; - th.runAndWait([&value]() { value = true;}); + th.runAndWait([&value]() { value = true; }); REQUIRE(value == true); } From 12c1f60cae191bf060194e32713add6d179d640b Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Mon, 16 Aug 2021 14:14:04 -0500 Subject: [PATCH 4/6] Removed thread_local from RunManagerMTWorker OscarMTProducer now guarantees the class methods are run on their own dedicated thread. --- .../interface/RunManagerMTWorker.h | 9 ++- .../Application/src/RunManagerMTWorker.cc | 70 +++---------------- 2 files changed, 15 insertions(+), 64 deletions(-) diff --git a/SimG4Core/Application/interface/RunManagerMTWorker.h b/SimG4Core/Application/interface/RunManagerMTWorker.h index f42895f449894..0442175117865 100644 --- a/SimG4Core/Application/interface/RunManagerMTWorker.h +++ b/SimG4Core/Application/interface/RunManagerMTWorker.h @@ -86,7 +86,8 @@ class RunManagerMTWorker { void DumpMagneticField(const G4Field*, const std::string&) const; - static void resetTLS(); + void resetTLS(); + int getThreadIndex() const { return m_thread_index; } Generator m_generator; edm::EDGetTokenT m_InToken; @@ -111,12 +112,14 @@ class RunManagerMTWorker { edm::ParameterSet m_p; struct TLSData; - static thread_local TLSData* m_tls; - static thread_local bool dumpMF; + TLSData* m_tls{nullptr}; + bool dumpMF{false}; G4SimEvent* m_simEvent; std::unique_ptr m_sVerbose; std::unordered_map> m_sdMakers; + + const int m_thread_index{-1}; }; #endif diff --git a/SimG4Core/Application/src/RunManagerMTWorker.cc b/SimG4Core/Application/src/RunManagerMTWorker.cc index 6e240cae85086..0f557b23a6cee 100644 --- a/SimG4Core/Application/src/RunManagerMTWorker.cc +++ b/SimG4Core/Application/src/RunManagerMTWorker.cc @@ -64,7 +64,6 @@ #include "tbb/task_arena.h" static std::once_flag applyOnce; -thread_local bool RunManagerMTWorker::dumpMF = false; // from https://hypernews.cern.ch/HyperNews/CMS/get/edmFramework/3302/2.html namespace { @@ -72,10 +71,6 @@ namespace { int get_new_thread_index() { return thread_counter++; } - thread_local int s_thread_index = get_new_thread_index(); - - int getThreadIndex() { return s_thread_index; } - void createWatchers(const edm::ParameterSet& iP, SimActivityRegistry* iReg, std::vector>& oWatchers, @@ -102,10 +97,6 @@ namespace { } } } - - std::atomic active_tlsdata{0}; - std::atomic tls_shutdown_timeout{false}; - std::atomic n_tls_shutdown_task{0}; } // namespace struct RunManagerMTWorker::TLSData { @@ -127,9 +118,9 @@ struct RunManagerMTWorker::TLSData { bool threadInitialized = false; bool runTerminated = false; - TLSData() { ++active_tlsdata; } + TLSData() {} - ~TLSData() { --active_tlsdata; } + ~TLSData() {} }; //This can not be a smart pointer since we must delete some of the members @@ -137,7 +128,7 @@ struct RunManagerMTWorker::TLSData { // other 'singletons' after those singletons have been deleted. Instead we // atempt to delete all TLS at RunManagerMTWorker destructor. If that fails for // some reason, it is better to leak than cause a crash. -thread_local RunManagerMTWorker::TLSData* RunManagerMTWorker::m_tls{nullptr}; +//thread_local RunManagerMTWorker::TLSData* RunManagerMTWorker::m_tls{nullptr}; RunManagerMTWorker::RunManagerMTWorker(const edm::ParameterSet& iConfig, edm::ConsumesCollector&& iC) : m_generator(iConfig.getParameter("Generator")), @@ -158,7 +149,8 @@ RunManagerMTWorker::RunManagerMTWorker(const edm::ParameterSet& iConfig, edm::Co m_pCustomUIsession(iConfig.getUntrackedParameter("CustomUIsession")), m_p(iConfig), m_simEvent(nullptr), - m_sVerbose(nullptr) { + m_sVerbose(nullptr), + m_thread_index{get_new_thread_index()} { std::vector onlySDs = iConfig.getParameter>("OnlySDs"); m_sdMakers = sim::sensitiveDetectorMakers(m_p, iC, onlySDs); std::vector watchers = iConfig.getParameter>("Watchers"); @@ -180,47 +172,9 @@ RunManagerMTWorker::RunManagerMTWorker(const edm::ParameterSet& iConfig, edm::Co edm::LogVerbatim("SimG4CoreApplication") << "SD[" << k << "] " << itr->first; } -RunManagerMTWorker::~RunManagerMTWorker() { - ++n_tls_shutdown_task; - resetTLS(); - - { - //make sure all tasks are done before continuing - timespec s; - s.tv_sec = 0; - s.tv_nsec = 10000; - while (n_tls_shutdown_task != 0) { - nanosleep(&s, nullptr); - } - } -} - -void RunManagerMTWorker::resetTLS() { - m_tls = nullptr; +RunManagerMTWorker::~RunManagerMTWorker() { resetTLS(); } - if (active_tlsdata != 0 and not tls_shutdown_timeout) { - ++n_tls_shutdown_task; - //need to run tasks on each thread which has set the tls - { - tbb::task_arena arena(tbb::task_arena::attach{}); - arena.enqueue([]() { RunManagerMTWorker::resetTLS(); }); - } - timespec s; - s.tv_sec = 0; - s.tv_nsec = 10000; - //we do not want this thread to be used for a new task since it - // has already cleared its structures. In order to fill all TBB - // threads we wait for all TLSes to clear - int count = 0; - while (active_tlsdata.load() != 0 and ++count < 1000) { - nanosleep(&s, nullptr); - } - if (count >= 1000) { - tls_shutdown_timeout = true; - } - } - --n_tls_shutdown_task; -} +void RunManagerMTWorker::resetTLS() { m_tls = nullptr; } void RunManagerMTWorker::beginRun(edm::EventSetup const& es) { for (auto& maker : m_sdMakers) { @@ -323,7 +277,7 @@ void RunManagerMTWorker::initializeG4(RunManagerMT* runManagerMaster, const edm: std::string fieldFile = m_p.getUntrackedParameter("FileNameField", ""); if (!fieldFile.empty()) { - std::call_once(applyOnce, []() { dumpMF = true; }); + std::call_once(applyOnce, [this]() { dumpMF = true; }); if (dumpMF) { edm::LogVerbatim("SimG4CoreApplication") << "RunManagerMTWorker: Dump magnetic field to file " << fieldFile; DumpMagneticField(tM->GetFieldManager()->GetDetectorField(), fieldFile); @@ -494,13 +448,7 @@ std::unique_ptr RunManagerMTWorker::produce(const edm::Event& inpevt // We have to do the per-thread initialization, and per-thread // per-run initialization here by ourselves. - if (nullptr == m_tls || !m_tls->threadInitialized) { - edm::LogVerbatim("SimG4CoreApplication") - << "RunManagerMTWorker::produce(): stream " << inpevt.streamID() << " thread " << getThreadIndex() - << " Geant4 initialisation for this thread"; - initializeG4(&runManagerMaster, es); - m_tls->threadInitialized = true; - } + assert(m_tls != nullptr and m_tls->threadInitialized); // Initialize run if (inpevt.id().run() != m_tls->currentRunNumber) { edm::LogVerbatim("SimG4CoreApplication") From e058dfbcd0a0c71771e1b76d99a1963951ca5b0c Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Mon, 16 Aug 2021 14:43:35 -0500 Subject: [PATCH 5/6] Be able to set worker thread stack size in configuration --- SimG4Core/Application/interface/ThreadHandoff.h | 2 +- SimG4Core/Application/plugins/OscarMTProducer.cc | 3 ++- SimG4Core/Application/src/ThreadHandoff.cc | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/SimG4Core/Application/interface/ThreadHandoff.h b/SimG4Core/Application/interface/ThreadHandoff.h index 9b9e0b9209090..e54aa32f6e8ab 100644 --- a/SimG4Core/Application/interface/ThreadHandoff.h +++ b/SimG4Core/Application/interface/ThreadHandoff.h @@ -32,7 +32,7 @@ namespace omt { class ThreadHandoff { public: - explicit ThreadHandoff(); + explicit ThreadHandoff(int stackSize); ~ThreadHandoff(); ThreadHandoff(const ThreadHandoff&) = delete; // stop default diff --git a/SimG4Core/Application/plugins/OscarMTProducer.cc b/SimG4Core/Application/plugins/OscarMTProducer.cc index 54d3668e45200..9f6a5bc2f1104 100644 --- a/SimG4Core/Application/plugins/OscarMTProducer.cc +++ b/SimG4Core/Application/plugins/OscarMTProducer.cc @@ -97,7 +97,8 @@ namespace { }; } // namespace -OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMasterThread* ms) { +OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMasterThread* ms) + : m_handoff{p.getUntrackedParameter("workerThreadStackSize", 10 * 1024 * 1024)} { // Random number generation not allowed here StaticRandomEngineSetUnset random(nullptr); diff --git a/SimG4Core/Application/src/ThreadHandoff.cc b/SimG4Core/Application/src/ThreadHandoff.cc index c4bab34e0354e..fda990f21f02b 100644 --- a/SimG4Core/Application/src/ThreadHandoff.cc +++ b/SimG4Core/Application/src/ThreadHandoff.cc @@ -37,14 +37,13 @@ namespace { // using namespace omt; -ThreadHandoff::ThreadHandoff() { +ThreadHandoff::ThreadHandoff(int stackSize) { pthread_attr_t attr; int erno; if (0 != (erno = pthread_attr_init(&attr))) { throw cms::Exception("ThreadInitFailed") << "Failed to initialize thread attributes (" << erno << ") " << errorMessage(erno); } - const int stackSize = 10 * 1024 * 1024; if (0 != (erno = pthread_attr_setstacksize(&attr, stackSize))) { throw cms::Exception("ThreadStackSizeFailed") From bb5bf396d025a0106c09868b6f6ea1c6134bc60e Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 17 Aug 2021 09:02:26 -0500 Subject: [PATCH 6/6] Set random number system in worker thread --- SimG4Core/Application/plugins/OscarMTProducer.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/SimG4Core/Application/plugins/OscarMTProducer.cc b/SimG4Core/Application/plugins/OscarMTProducer.cc index 9f6a5bc2f1104..d33504f82f66e 100644 --- a/SimG4Core/Application/plugins/OscarMTProducer.cc +++ b/SimG4Core/Application/plugins/OscarMTProducer.cc @@ -91,6 +91,8 @@ namespace { explicit StaticRandomEngineSetUnset(CLHEP::HepRandomEngine* engine); ~StaticRandomEngineSetUnset(); + CLHEP::HepRandomEngine* currentEngine() { return m_currentEngine; } + private: CLHEP::HepRandomEngine* m_currentEngine; CLHEP::HepRandomEngine* m_previousEngine; @@ -105,6 +107,7 @@ OscarMTProducer::OscarMTProducer(edm::ParameterSet const& p, const OscarMTMaster auto token = edm::ServiceRegistry::instance().presentToken(); m_handoff.runAndWait([this, &p, token]() { edm::ServiceRegistry::Operate guard{token}; + StaticRandomEngineSetUnset random(nullptr); m_runManagerWorker = std::make_unique(p, consumesCollector()); }); m_masterThread = ms; @@ -228,6 +231,7 @@ void OscarMTProducer::endRun(const edm::Run&, const edm::EventSetup&) { edm::LogVerbatim("SimG4CoreApplication") << "OscarMTProducer::endRun"; auto token = edm::ServiceRegistry::instance().presentToken(); m_handoff.runAndWait([this, token]() { + StaticRandomEngineSetUnset random(nullptr); edm::ServiceRegistry::Operate guard{token}; m_runManagerWorker->endRun(); }); @@ -236,6 +240,7 @@ void OscarMTProducer::endRun(const edm::Run&, const edm::EventSetup&) { void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) { StaticRandomEngineSetUnset random(e.streamID()); + auto engine = random.currentEngine(); edm::LogVerbatim("SimG4CoreApplication") << "Produce event " << e.id() << " stream " << e.streamID(); //edm::LogVerbatim("SimG4CoreApplication") << " rand= " << G4UniformRand(); @@ -245,8 +250,9 @@ void OscarMTProducer::produce(edm::Event& e, const edm::EventSetup& es) { std::unique_ptr evt; try { auto token = edm::ServiceRegistry::instance().presentToken(); - m_handoff.runAndWait([this, &e, &es, &evt, token]() { + m_handoff.runAndWait([this, &e, &es, &evt, token, engine]() { edm::ServiceRegistry::Operate guard{token}; + StaticRandomEngineSetUnset random(engine); evt = m_runManagerWorker->produce(e, es, globalCache()->runManagerMaster()); }); } catch (const SimG4Exception& simg4ex) {