Major DQM update needed for the online

DQM/HLXMonitor/src/HLXMonitor.cc

@@ -824,7 +824,8 @@ void HLXMonitor::SaveDQMFile(){
                      "^(Reference/)?([^/]+)",
                      rewrite,
                      0,
-		                 (DQMStore::SaveReferenceTag)saveReference_, saveReferenceQMin_);
+		     0,
+		     (DQMStore::SaveReferenceTag)saveReference_, saveReferenceQMin_);
 
   //dbe_->save(tempStreamer.str());
 }

DQMServices/Components/BuildFile.xml

@@ -3,6 +3,7 @@
 <use   name="DataFormats/Scalers"/>
 <use   name="DataFormats/L1GlobalTrigger"/>
 <use   name="DQMServices/ClientConfig"/>
+<use   name="EventFilter/Utilities"/>
 <use   name="DQMServices/Core"/>
 <use   name="FWCore/Catalog"/>
 <use   name="FWCore/Framework"/>

DQMServices/Components/bin/fastHadd.cc

@@ -93,6 +93,7 @@ PATH=/afs/cern.ch/work/r/rovere/protocolbuf/bin
 #include <TFile.h>
 #include <TBufferFile.h>
 #include <TObject.h>
+#include <TObjString.h>
 #include <TH1.h>
 #include <TKey.h>
 
@@ -112,19 +113,37 @@ static bool lessThanMME(const std::string &lhs_dirname,
 struct MicroME {
   MicroME(const std::string * full,
           const std::string * dir,
-          const std::string * obj)
-      :fullname(full), dirname(dir), objname(obj) {}
+          const std::string * obj,
+          uint32_t flags = 0)
+      :fullname(full), dirname(dir), objname(obj), flags(flags) {}
   const std::string * fullname;
   const std::string * dirname;
   const std::string * objname;
-  TH1 * obj;
+  mutable TObject * obj;
+
+  uint32_t flags;
 
   bool operator<(const MicroME &rhs) const {
     return lessThanMME(*this->dirname,
                        *this->objname,
                        *rhs.dirname,
                        *rhs.objname);
-  }
+  };
+
+  void add(TObject *obj_to_add) const {
+      DEBUG(1, "Merging: " << obj->GetName() <<
+        " << " << obj_to_add->GetName() << std::endl);
+
+      if (dynamic_cast<TH1 *>(obj) && dynamic_cast<TH1 *>(obj_to_add)) {
+        dynamic_cast<TH1 *>(obj)->Add(dynamic_cast<TH1 *>(obj_to_add));
+      } else if (dynamic_cast<TObjString *>(obj) && dynamic_cast<TObjString *>(obj_to_add)) {
+
+
+      } else {
+        DEBUG(1, "Cannot merge (different types): " << obj->GetName() <<
+          " << " << obj_to_add->GetName() << std::endl);
+      }
+  };
 };
 
 enum TaskType {
@@ -159,7 +178,8 @@ inline TObject * extractNextObject(TBufferFile &buf) {
 static void get_info(const dqmstorepb::ROOTFilePB::Histo &h,
                      std::string &dirname,
                      std::string &objname,
-                     TH1 ** obj) {
+                     TObject ** obj) {
+
   size_t slash = h.full_pathname().rfind('/');
   size_t dirpos = (slash == std::string::npos ? 0 : slash);
   size_t namepos = (slash == std::string::npos ? 0 : slash+1);
@@ -169,7 +189,7 @@ static void get_info(const dqmstorepb::ROOTFilePB::Histo &h,
                   (void*)h.streamed_histo().data(),
                   kFALSE);
   buf.Reset();
-  *obj = static_cast<TH1*>(extractNextObject(buf));
+  *obj = extractNextObject(buf);
   if (!*obj) {
     std::cerr << "Error reading element: " << h.full_pathname() << std::endl;
   }
@@ -207,6 +227,7 @@ void fillMessage(dqmstorepb::ROOTFilePB &dqmstore_output_msg,
     TBufferFile buffer(TBufferFile::kWrite);
     buffer.WriteObject(mi->obj);
     h->set_size(buffer.Length());
+    h->set_flags(mi->flags);
     h->set_streamed_histo((const void*)buffer.Buffer(),
                           buffer.Length());
     delete mi->obj;
@@ -234,16 +255,19 @@ void processDirectory(TFile *file,
         subdir += '/';
       subdir += obj->GetName();
       processDirectory(file, subdir, dirs, objs, fullnames, micromes);
-    } else if (dynamic_cast<TH1 *>(obj)) {
-      (dynamic_cast<TH1*>(obj))->SetDirectory(0);
+    } else if ((dynamic_cast<TH1 *>(obj)) && (dynamic_cast<TObjString *>(obj))) {
+      if (dynamic_cast<TH1 *>(obj)) {
+        dynamic_cast<TH1 *>(obj)->SetDirectory(0);
+      }
+
       DEBUG(2, curdir << "/" << obj->GetName() << "\n");
       MicroME mme(&*(fullnames.insert(curdir
                                       + '/'
                                       + std::string(obj->GetName())).first),
                      &*(dirs.insert(curdir).first),
                      &*(objs.insert(obj->GetName()).first));
       if (obj) {
-        mme.obj = dynamic_cast<TH1*>(obj);
+        mme.obj = obj;
         micromes.insert(mme);
       }
     }
@@ -295,7 +319,7 @@ int convertFile(const std::string &output_filename,
                     (void*)h.streamed_histo().data(),
                     kFALSE);
     buf.Reset();
-    TH1 *obj = static_cast<TH1*>(extractNextObject(buf));
+    TObject *obj = static_cast<TH1*>(extractNextObject(buf));
     std::string path,objname;
     get_info(h, path, objname, &obj);
     gDirectory->cd("/");
@@ -354,6 +378,7 @@ int dumpFiles(const std::vector<std::string> &filenames) {
       buf.Reset();
       TObject *obj = extractNextObject(buf);
       DEBUG(1, obj->GetName() << std::endl);
+      DEBUG(1, "Flags: " << h.flags() << std::endl);
     }
   }
   google::protobuf::ShutdownProtobufLibrary();
@@ -391,12 +416,13 @@ int addFiles(const std::string &output_filename,
     for (int i = 0; i < dqmstore_message.histo_size(); i++) {
       std::string path;
       std::string objname;
-      TH1 *obj = NULL;
+      TObject *obj = NULL;
       const dqmstorepb::ROOTFilePB::Histo &h = dqmstore_message.histo(i);
       get_info(h, path, objname, &obj);
       MicroME * mme = new MicroME(&*(fullnames.insert(h.full_pathname()).first),
                                   &*(dirs.insert(path).first),
-                                  &*(objs.insert(objname).first));
+                                  &*(objs.insert(objname).first),
+                                  h.flags());
       if (obj) {
         mme->obj = obj;
         micromes.insert(*mme);
@@ -431,17 +457,18 @@ int addFiles(const std::string &output_filename,
       std::string path;
       std::string objname;
       dqmstorepb::ROOTFilePB::Histo h;
-      TH1 *obj = NULL;
+      TObject *obj = NULL;
       if (elem < dqmstore_msg.histo_size()) {
         dqmstorepb::ROOTFilePB::Histo &h =
             const_cast<dqmstorepb::ROOTFilePB::Histo &>(dqmstore_msg.histo(elem));
         get_info(h, path, objname, &obj);
+
         DEBUG(2, "Comparing " << *(*mi).dirname << "/"
               << *(*mi).objname << " vs "
               << h.full_pathname() << std::endl);
         int diff = (*mi).fullname->compare(h.full_pathname());
         if (diff == 0 && obj != NULL) {
-          (*mi).obj->Add(obj);
+          mi->add(obj);
           delete obj;
           ++elem;
         } else if (! lessThanMME(*(*mi).dirname, *(*mi).objname,
@@ -482,7 +509,7 @@ int addFiles(const std::string &output_filename,
     while (elem < dqmstore_msg.histo_size()) {
       std::string path;
       std::string objname;
-      TH1 *obj = NULL;
+      TObject *obj = NULL;
 
       const dqmstorepb::ROOTFilePB::Histo &h = dqmstore_msg.histo(elem);
       get_info(h, path, objname, &obj);
@@ -491,7 +518,7 @@ int addFiles(const std::string &output_filename,
                                   &*(dirs.insert(path).first),
                                   &*(objs.insert(objname).first));
       if (obj) {
-        mme->obj = const_cast<TH1*>(obj);
+        mme->obj = obj;
         micromes.insert(*mme);
         ++elem;
       }

DQMServices/Components/plugins/MEtoEDMConverter.cc

@@ -26,7 +26,7 @@ MEtoEDMConverter::MEtoEDMConverter(const edm::ParameterSet & iPSet) :
   frequency = iPSet.getUntrackedParameter<int>("Frequency",50);
   path = iPSet.getUntrackedParameter<std::string>("MEPathToSave");
   deleteAfterCopy = iPSet.getUntrackedParameter<bool>("deleteAfterCopy",false);
-  enableMultiThread_ = iPSet.getUntrackedParameter<bool>("enableMultiThread",false);
+  enableMultiThread_ = false;
   // use value of first digit to determine default output level (inclusive)
   // 0 is none, 1 is basic, 2 is fill output, 3 is gather output
   verbosity %= 10;
@@ -46,6 +46,7 @@ MEtoEDMConverter::MEtoEDMConverter(const edm::ParameterSet & iPSet) :
   // get dqm info
   dbe = 0;
   dbe = edm::Service<DQMStore>().operator->();
+  enableMultiThread_ = dbe->enableMultiThread_;
 
   std::string sName;
 

DQMServices/Components/python/DQMFileSaver_cfi.py

@@ -18,19 +18,9 @@
     version = cms.untracked.int32(1),
     # runIsComplete
     runIsComplete = cms.untracked.bool(False),
-    # Enable MultiThread behaviour, i.e., save only MonitorElements
-    # indexed by the current Run
-    enableMultiThread = cms.untracked.bool(False),
 
     # Save file every N lumi sections (-1: disabled)
     saveByLumiSection = cms.untracked.int32(-1),
-    # Save file every N events (-1: disabled)
-    saveByEvent = cms.untracked.int32(-1),
-    # Save file every N minutes (-1: disabled)
-    saveByMinute = cms.untracked.int32(-1),
-    # Save file every 2**N minutes until saveByTime > saveByMinute
-    saveByTime = cms.untracked.int32(-1),
-
     # Save file every N runs (-1: disabled)
     saveByRun = cms.untracked.int32(-1),
     # Save file at the end of the job

DQMServices/Components/python/test/create_filePB_cfg.py

@@ -0,0 +1,79 @@
+import FWCore.ParameterSet.Config as cms
+import DQMServices.Components.test.checkBooking as booking
+import DQMServices.Components.test.createElements as c
+import sys
+
+process = cms.Process("TEST")
+
+# load DQM
+process.load("DQMServices.Core.DQM_cfg")
+process.load("DQMServices.Components.DQMEnvironment_cfi")
+
+
+b = booking.BookingParams(sys.argv)
+b.doCheck(testOnly=False)
+
+process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(10))
+process.source = cms.Source("EmptySource",
+                            firstLuminosityBlock = cms.untracked.uint32(1),
+                            firstEvent = cms.untracked.uint32(1),
+                            numberEventsInLuminosityBlock = cms.untracked.uint32(1))
+
+
+elements = c.createElements()
+readRunElements = c.createReadRunElements()
+readLumiElements = c.createReadLumiElements()
+
+process.filler = cms.EDAnalyzer("DummyBookFillDQMStore" + b.mt_postfix(),
+                                folder    = cms.untracked.string("TestFolder/"),
+                                elements  = cms.untracked.VPSet(*elements),
+                                fillRuns  = cms.untracked.bool(True),
+                                fillLumis = cms.untracked.bool(True),
+                                book_at_constructor = cms.untracked.bool(b.getBookLogic('CTOR')),
+                                book_at_beginJob = cms.untracked.bool(b.getBookLogic('BJ')),
+                                book_at_beginRun = cms.untracked.bool(b.getBookLogic('BR')))
+
+process.out = cms.OutputModule("DQMRootOutputModule",
+                               fileName = cms.untracked.string("dqm_file4.root"))
+
+
+process.p = cms.Path(process.filler)
+process.dqmsave_step = cms.Path(process.dqmSaver)
+process.o = cms.EndPath(process.out)
+
+process.schedule = cms.Schedule(
+    process.p,
+    process.dqmsave_step
+)
+
+process.add_(cms.Service("DQMStore"))
+
+if b.multithread():
+    process.out.enableMultiThread = cms.untracked.bool(True)
+    process.DQMStore.enableMultiThread = cms.untracked.bool(True)
+    process.options = cms.untracked.PSet(
+        numberOfThreads = cms.untracked.uint32(4),
+        numberOfStreams = cms.untracked.uint32(4)
+        )
+
+
+
+
+if len(sys.argv) > 3:
+    if sys.argv[3] == "ForceReset":
+        print "Forcing Reset of histograms at every Run Transition."
+        process.DQMStore.forceResetOnBeginRun = cms.untracked.bool(True)
+
+
+#----------------------------------------------------------#
+### global options Online ###
+process.DQMStore.LSbasedMode = cms.untracked.bool(True)
+process.DQMStore.verbose = cms.untracked.int32(5)
+
+process.dqmSaver.workflow = ''
+process.dqmSaver.convention = 'FilterUnit'
+process.dqmSaver.saveByLumiSection = True
+process.dqmSaver.fileFormat = cms.untracked.string('PB')
+process.dqmSaver.fakeFilterUnitMode = cms.untracked.bool(True)
+
+#process.add_(cms.Service("Tracer"))

DQMServices/Components/python/test/create_filePB_ref_cfg.py

@@ -0,0 +1,78 @@
+import FWCore.ParameterSet.Config as cms
+import DQMServices.Components.test.checkBooking as booking
+import DQMServices.Components.test.createElements as c
+import sys
+
+process = cms.Process("TEST")
+
+# load DQM
+process.load("DQMServices.Core.DQM_cfg")
+process.load("DQMServices.Components.DQMEnvironment_cfi")
+
+
+b = booking.BookingParams(sys.argv)
+b.doCheck(testOnly=False)
+
+process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(10))
+process.source = cms.Source("EmptySource",
+                            firstLuminosityBlock = cms.untracked.uint32(1),
+                            firstEvent = cms.untracked.uint32(1),
+                            numberEventsInLuminosityBlock = cms.untracked.uint32(1))
+
+
+elements = c.createElements()
+readRunElements = c.createReadRunElements()
+readLumiElements = c.createReadLumiElements()
+
+process.filler = cms.EDAnalyzer("DummyBookFillDQMStore" + b.mt_postfix(),
+                                folder    = cms.untracked.string("TestFolder/"),
+                                elements  = cms.untracked.VPSet(*elements),
+                                fillRuns  = cms.untracked.bool(True),
+                                fillLumis = cms.untracked.bool(True),
+                                book_at_constructor = cms.untracked.bool(b.getBookLogic('CTOR')),
+                                book_at_beginJob = cms.untracked.bool(b.getBookLogic('BJ')),
+                                book_at_beginRun = cms.untracked.bool(b.getBookLogic('BR')))
+
+process.out = cms.OutputModule("DQMRootOutputModule",
+                               fileName = cms.untracked.string("dqm_filePB_ref.root"))
+
+
+process.p = cms.Path(process.filler)
+process.o = cms.EndPath(process.out)
+
+process.schedule = cms.Schedule(
+    process.p,
+    process.o
+)
+
+process.add_(cms.Service("DQMStore"))
+
+if b.multithread():
+    process.out.enableMultiThread = cms.untracked.bool(True)
+    process.DQMStore.enableMultiThread = cms.untracked.bool(True)
+    process.options = cms.untracked.PSet(
+        numberOfThreads = cms.untracked.uint32(4),
+        numberOfStreams = cms.untracked.uint32(4)
+        )
+
+
+
+
+if len(sys.argv) > 3:
+    if sys.argv[3] == "ForceReset":
+        print "Forcing Reset of histograms at every Run Transition."
+        process.DQMStore.forceResetOnBeginRun = cms.untracked.bool(True)
+
+
+#----------------------------------------------------------#                                                                                                                                      
+### global options Online ###                                                                                                                                                                       
+process.DQMStore.LSbasedMode = cms.untracked.bool(False)
+process.DQMStore.verbose = cms.untracked.int32(5)
+
+process.dqmSaver.workflow = ''
+process.dqmSaver.convention = 'FilterUnit'
+process.dqmSaver.saveByLumiSection = -1
+process.dqmSaver.fileFormat = cms.untracked.string('ROOT')
+process.dqmSaver.fakeFilterUnitMode = cms.untracked.bool(True)
+
+#process.add_(cms.Service("Tracer"))