Decrease memory used by OscarMTProducer

SimG4CMS/Calo/interface/CaloSD.h

@@ -27,6 +27,7 @@
 
 #include <vector>
 #include <map>
+#include <memory>
 
 class G4Step;
 class G4HCofThisEvent;
@@ -64,6 +65,7 @@ class CaloSD : public SensitiveCaloDetector,
 
   void clearHits() override;
   void fillHits(edm::PCaloHitContainer&, const std::string&) override;
+  void reset() override;
 
 protected:
   virtual double getEnergyDeposit(const G4Step* step);
@@ -163,10 +165,7 @@ class CaloSD : public SensitiveCaloDetector,
 
   std::map<CaloHitID, CaloG4Hit*> hitMap;
   std::map<int, TrackWithHistory*> tkMap;
-
-  std::vector<CaloG4Hit*> reusehit;
-  std::vector<CaloG4Hit*> hitvec;
-  std::vector<unsigned int> selIndex;
+  std::vector<std::unique_ptr<CaloG4Hit>> reusehit;
 };
 
 #endif  // SimG4CMS_CaloSD_h

SimG4CMS/Calo/src/CaloSD.cc

@@ -20,7 +20,6 @@
 #include "G4SystemOfUnits.hh"
 #include "G4PhysicalConstants.hh"
 
-#include <iostream>
 #include <fstream>
 
 //#define EDM_ML_DEBUG
@@ -103,7 +102,7 @@ CaloSD::CaloSD(const std::string& name,
                               << "        Time Slice Unit " << timeSlice << " Ignore TrackID Flag " << ignoreTrackID;
 }
 
-CaloSD::~CaloSD() { delete theHC; }
+CaloSD::~CaloSD() { }
 
 G4bool CaloSD::ProcessHits(G4Step* aStep, G4TouchableHistory*) {
   NaNTrap(aStep);
@@ -260,6 +259,7 @@ void CaloSD::Initialize(G4HCofThisEvent* HCE) {
   if (hcID < 0) {
     hcID = G4SDManager::GetSDMpointer()->GetCollectionID(collectionName[0]);
   }
+  //theHC ownership is transfered here to HCE
   HCE->AddHitsCollection(hcID, theHC);
 }
 
@@ -360,10 +360,10 @@ CaloG4Hit* CaloSD::createNewHit(const G4Step* aStep) {
 
   CaloG4Hit* aHit;
   if (!reusehit.empty()) {
-    aHit = reusehit[0];
+    aHit = reusehit.back().release();
     aHit->setEM(0.f);
     aHit->setHadr(0.f);
-    reusehit.erase(reusehit.begin());
+    reusehit.pop_back();
   } else {
     aHit = new CaloG4Hit;
   }
@@ -544,15 +544,12 @@ void CaloSD::update(const ::EndOfEvent*) {
                               << " E0mean= " << ee << " Zglob= " << zglob << " Zloc= " << zloc << " ";
 
   tkMap.erase(tkMap.begin(), tkMap.end());
+  std::vector<std::unique_ptr<CaloG4Hit>>().swap(reusehit);
+  if (useMap) hitMap.erase (hitMap.begin(), hitMap.end());
 }
 
 void CaloSD::clearHits() {
-  if (useMap)
-    hitMap.erase(hitMap.begin(), hitMap.end());
-  for (unsigned int i = 0; i < reusehit.size(); ++i)
-    delete reusehit[i];
-  std::vector<CaloG4Hit*>().swap(reusehit);
-  cleanIndex = 0;
+  cleanIndex  = 0;
   previousID.reset();
   primIDSaved = -99;
 #ifdef EDM_ML_DEBUG
@@ -562,6 +559,12 @@ void CaloSD::clearHits() {
   slave.get()->Initialize();
 }
 
+void CaloSD::reset() {
+  if(fpCaloG4HitAllocator) {
+    fpCaloG4HitAllocator->ResetStorage();
+  }
+}
+
 void CaloSD::initRun() {}
 
 void CaloSD::initEvent(const BeginOfEvent*) {}
@@ -699,13 +702,13 @@ void CaloSD::cleanHitCollection() {
 #ifdef EDM_ML_DEBUG
   edm::LogVerbatim("CaloSim") << "CaloSD: collection before merging, size = " << theHC->entries();
 #endif
-
-  selIndex.reserve(theHC->entries() - cleanIndex);
-  if (reusehit.empty())
-    reusehit.reserve(theHC->entries() - cleanIndex);
+  if (reusehit.empty()) 
+    reusehit.reserve(theHC->entries() - cleanIndex); 
 
   // if no map used, merge before hits to have the save situation as a map
-  if (!useMap) {
+  if ( !useMap ) {
+    std::vector<CaloG4Hit*> hitvec;
+
     hitvec.swap(*theCollection);
     sort((hitvec.begin() + cleanIndex), hitvec.end(), CaloG4HitLess());
 #ifdef EDM_ML_DEBUG
@@ -714,18 +717,17 @@ void CaloSD::cleanHitCollection() {
     for (unsigned int i = 0; i < hitvec.size(); ++i)
       edm::LogVerbatim("CaloSim") << i << " " << *hitvec[i];
 #endif
-    unsigned int i, j;
     CaloG4HitEqual equal;
-    for (i = cleanIndex; i < hitvec.size(); ++i) {
-      selIndex.push_back(i - cleanIndex);
+    for (unsigned int i= cleanIndex; i < hitvec.size(); ++i) {
       int jump = 0;
-      for (j = i + 1; j < hitvec.size() && equal(hitvec[i], hitvec[j]); ++j) {
+      for (unsigned int j = i + 1; j < hitvec.size() && equal(hitvec[i], hitvec[j]); ++j) {
         ++jump;
         // merge j to i
         (*hitvec[i]).addEnergyDeposit(*hitvec[j]);
         (*hitvec[j]).setEM(0.);
         (*hitvec[j]).setHadr(0.);
-        reusehit.push_back(hitvec[j]);
+        reusehit.emplace_back(hitvec[j]);
+        hitvec[j] = nullptr;
       }
       i += jump;
     }
@@ -734,19 +736,17 @@ void CaloSD::cleanHitCollection() {
     for (unsigned int i = 0; i < hitvec.size(); ++i)
       edm::LogVerbatim("CaloSim") << i << " " << *hitvec[i];
 #endif
-    for (unsigned int i = cleanIndex; i < cleanIndex + selIndex.size(); ++i) {
-      hitvec[i] = hitvec[selIndex[i - cleanIndex] + cleanIndex];
-    }
-    hitvec.resize(cleanIndex + selIndex.size());
+    //move all nullptr to end of list and then remove them
+    hitvec.erase(std::stable_partition(hitvec.begin()+cleanIndex, 
+                                       hitvec.end(), [](auto* p) { return p!= nullptr;}),
+                 hitvec.end());
 #ifdef EDM_ML_DEBUG
     edm::LogVerbatim("CaloSim") << "CaloSD::cleanHitCollection: remove the merged hits in buffer,"
                                 << " new size = " << hitvec.size();
-    for (unsigned int i = 0; i < hitvec.size(); ++i)
+    for (unsigned int i = 0; i < hitvec.size(); ++i) 
       edm::LogVerbatim("CaloSim") << i << " " << *hitvec[i];
 #endif
     hitvec.swap(*theCollection);
-    std::vector<CaloG4Hit*>().swap(hitvec);
-    selIndex.clear();
     totalHits = theHC->entries();
   }
 
@@ -757,8 +757,7 @@ void CaloSD::cleanHitCollection() {
 #endif
 
   int addhit = 0;
-  selIndex.reserve(theCollection->size() - cleanIndex);
-  for (unsigned int i = cleanIndex; i < theCollection->size(); ++i) {
+  for (unsigned int i = cleanIndex; i < theCollection->size(); ++i) {   
     CaloG4Hit* aHit((*theCollection)[i]);
 
     // selection
@@ -773,11 +772,10 @@ void CaloSD::cleanHitCollection() {
 #endif
 
       // create the list of hits to be reused
-
-      reusehit.push_back((*theCollection)[i]);
+
+      reusehit.emplace_back((*theCollection)[i]);
+      (*theCollection)[i] = nullptr;
       ++addhit;
-    } else {
-      selIndex.push_back(i - cleanIndex);
     }
   }
 
@@ -794,13 +792,11 @@ void CaloSD::cleanHitCollection() {
       }
     }
   }
-  for (unsigned int j = 0; j < selIndex.size(); ++j) {
-    (*theCollection)[cleanIndex + j] = (*theCollection)[cleanIndex + selIndex[j]];
-  }
-
-  theCollection->resize(cleanIndex + selIndex.size());
-  std::vector<unsigned int>().swap(selIndex);
 
+  //move all nullptr to end of list and then remove them
+  theCollection->erase(std::stable_partition(theCollection->begin()+cleanIndex,
+                                             theCollection->end(), [](auto* p) { return p!= nullptr;}),
+                       theCollection->end());
 #ifdef EDM_ML_DEBUG
   edm::LogVerbatim("CaloSim") << "CaloSD: hit collection after selection, size = " << theHC->entries();
   theHC->PrintAllHits();

SimG4CMS/Tracker/interface/TkAccumulatingSensitiveDetector.h

@@ -64,7 +64,7 @@ public Observer<const BeginOfJob*>
     std::unique_ptr<TrackingSlaveSD> slaveHighTof;
     std::unique_ptr<FrameRotation>   theRotation;
     std::unique_ptr<const G4ProcessTypeEnumerator> theG4ProcTypeEnumerator;
-    std::unique_ptr<TrackerG4SimHitNumberingScheme> theNumberingScheme;
+    TrackerG4SimHitNumberingScheme* theNumberingScheme; //does not own
     bool allowZeroEnergyLoss;
     bool printHits;
     bool neverAccumulate;

SimG4CMS/Tracker/src/TkAccumulatingSensitiveDetector.cc

@@ -93,7 +93,7 @@ TkAccumulatingSensitiveDetector::TkAccumulatingSensitiveDetector(const std::stri
   setNames(temp);  
 
   theG4ProcTypeEnumerator.reset(new G4ProcessTypeEnumerator);
-  theNumberingScheme.reset(nullptr);
+  theNumberingScheme = nullptr;
 }
 
 TkAccumulatingSensitiveDetector::~TkAccumulatingSensitiveDetector() 
@@ -126,7 +126,7 @@ bool TkAccumulatingSensitiveDetector::ProcessHits(G4Step * aStep, G4TouchableHis
 
 uint32_t TkAccumulatingSensitiveDetector::setDetUnitId(const G4Step * step)
 { 
-  return theNumberingScheme.get()->g4ToNumberingScheme(step->GetPreStepPoint()->GetTouchable());
+  return theNumberingScheme->g4ToNumberingScheme(step->GetPreStepPoint()->GetTouchable());
 }
 
 void TkAccumulatingSensitiveDetector::update(const BeginOfTrack *bot){
@@ -369,7 +369,7 @@ void TkAccumulatingSensitiveDetector::update(const BeginOfJob * i)
     edm::ESTransientHandle<DDCompactView> pView;
     es->get<IdealGeometryRecord>().get(pView);
 
-    theNumberingScheme.reset(&(numberingScheme(*pView,*pDD)));
+    theNumberingScheme = &(numberingScheme(*pView,*pDD));
 }
 
 void TkAccumulatingSensitiveDetector::clearHits()

SimG4Core/Application/interface/RunManager.h

@@ -139,7 +139,6 @@ class RunManager
   std::vector<std::string> m_G4Commands;
   edm::ParameterSet m_p;
 
-  AttachSD * m_attach;
   std::vector<SensitiveTkDetector*> m_sensTkDets;
   std::vector<SensitiveCaloDetector*> m_sensCaloDets;
 

SimG4Core/Application/interface/RunManagerMTWorker.h

@@ -8,6 +8,7 @@
 #include "SimDataFormats/Forward/interface/LHCTransportLinkContainer.h"
 
 #include <memory>
+#include <tbb/concurrent_vector.h>
 
 namespace edm {
   class ParameterSet;
@@ -43,12 +44,13 @@ class RunManagerMTWorker {
 
   void endRun();
 
-  void produce(const edm::Event& inpevt, const edm::EventSetup& es, RunManagerMT& runManagerMaster);
+  std::unique_ptr<G4SimEvent> 
+    produce(const edm::Event& inpevt, const edm::EventSetup& es, RunManagerMT& runManagerMaster);
 
   void abortEvent();
   void abortRun(bool softAbort=false);
 
-  inline G4SimEvent * simEvent() { return m_simEvent.get(); }
+  inline G4SimEvent * simEvent() { return m_simEvent; }
 
   void Connect(RunAction*);
   void Connect(EventAction*);
@@ -91,9 +93,10 @@ class RunManagerMTWorker {
   edm::ParameterSet m_p;
 
   struct TLSData;
-  static thread_local TLSData *m_tls;
+  static thread_local TLSData* m_tls;
+  static void resetTLS();
 
-  std::unique_ptr<G4SimEvent> m_simEvent;
+  G4SimEvent* m_simEvent;
   std::unique_ptr<CMSSteppingVerbose> m_sVerbose;
 };
 

SimG4Core/Application/plugins/OscarMTProducer.cc

@@ -172,12 +172,11 @@ void OscarMTProducer::produce(edm::Event & e, const edm::EventSetup & es)
     << "Produce event " << e.id() << " stream " << e.streamID() 
     << " rand= " << G4UniformRand();
 
-  std::vector<SensitiveTkDetector*>& sTk =
-    m_runManagerWorker->sensTkDetectors();
-  std::vector<SensitiveCaloDetector*>& sCalo =
-    m_runManagerWorker->sensCaloDetectors();
+  auto& sTk = m_runManagerWorker->sensTkDetectors();
+  auto& sCalo = m_runManagerWorker->sensCaloDetectors();
 
-  try { m_runManagerWorker->produce(e, es, globalCache()->runManagerMaster()); }
+  std::unique_ptr<G4SimEvent> evt;
+  try { evt=m_runManagerWorker->produce(e, es, globalCache()->runManagerMaster()); }
   catch ( const SimG4Exception& simg4ex ) {
 
     edm::LogWarning("SimG4CoreApplication") << "SimG4Exception caght! " 
@@ -193,7 +192,6 @@ void OscarMTProducer::produce(edm::Event & e, const edm::EventSetup & es)
     p1(new edm::SimTrackContainer);
   std::unique_ptr<edm::SimVertexContainer>
     p2(new edm::SimVertexContainer);
-  G4SimEvent * evt = m_runManagerWorker->simEvent();
   evt->load(*p1);
   evt->load(*p2);   
 

SimG4Core/Application/plugins/OscarProducer.cc

@@ -157,9 +157,9 @@ void OscarProducer::produce(edm::Event & e, const edm::EventSetup & es)
 {
   StaticRandomEngineSetUnset random(e.streamID());
 
-  std::vector<SensitiveTkDetector*>& sTk = 
+  auto& sTk = 
     m_runManager->sensTkDetectors();
-  std::vector<SensitiveCaloDetector*>& sCalo =
+  auto& sCalo =
     m_runManager->sensCaloDetectors();
 
   try { m_runManager->produce(e, es); }

SimG4Core/Application/src/RunManager.cc

@@ -148,7 +148,6 @@ RunManager::RunManager(edm::ParameterSet const & p, edm::ConsumesCollector&& iC)
 
   m_physicsList.reset(nullptr);
   m_prodCuts.reset(nullptr);
-  m_attach = nullptr;
 
   m_check = p.getUntrackedParameter<bool>("CheckOverlap",false);
   m_WriteFile = p.getUntrackedParameter<std::string>("FileNameGDML","");
@@ -237,15 +236,16 @@ void RunManager::initG4(const edm::EventSetup & es)
   // we need the track manager now
   m_trackManager = std::unique_ptr<SimTrackManager>(new SimTrackManager);
 
-  // attach sensitive detector
-  m_attach = new AttachSD;
+  {
+    // attach sensitive detector
 
-  std::pair< std::vector<SensitiveTkDetector*>,
-    std::vector<SensitiveCaloDetector*> > sensDets = 
-    m_attach->create((*pDD),catalog_,m_p,m_trackManager.get(),m_registry);
+    AttachSD attach;
+    auto sensDets = 
+      attach.create((*pDD),catalog_,m_p,m_trackManager.get(),m_registry);
 
-  m_sensTkDets.swap(sensDets.first);
-  m_sensCaloDets.swap(sensDets.second);
+    m_sensTkDets.swap(sensDets.first);
+    m_sensCaloDets.swap(sensDets.second);
+  }
 
   edm::LogInfo("SimG4CoreApplication") 
     << " RunManager: Sensitive Detector "