Introduce a work-around for a mysterious InputSource bug (DQMProtobuf…

…Reader). Also implement a way to load json files which use unexpected 'data' field format in DQMProtobufReader.
dmitrijus · Aug 20, 2014 · fc0dbea · fc0dbea
1 parent e5b1fe9
commit fc0dbea
Show file tree

Hide file tree

Showing 6 changed files with 35 additions and 24 deletions.
diff --git a/DQMServices/StreamerIO/plugins/DQMFileIterator.cc b/DQMServices/StreamerIO/plugins/DQMFileIterator.cc
@@ -17,7 +17,7 @@
 namespace dqmservices {
 
 DQMFileIterator::LumiEntry DQMFileIterator::LumiEntry::load_json(
-    const std::string& filename, int lumiNumber, JsonType type) {
+    const std::string& filename, int lumiNumber, unsigned int datafn_position) {
   boost::property_tree::ptree pt;
   read_json(filename, pt);
 
@@ -29,14 +29,8 @@ DQMFileIterator::LumiEntry DQMFileIterator::LumiEntry::load_json(
                       ->second.get_value<std::size_t>();
 
   lumi.ls = lumiNumber;
-
-  if (type == JS_PROTOBUF) {
-    lumi.datafilename = std::next(pt.get_child("data").begin(), 4)
-                            ->second.get_value<std::string>();
-  } else {
-    lumi.datafilename = std::next(pt.get_child("data").begin(), 3)
-                            ->second.get_value<std::string>();
-  }
+  lumi.datafilename = std::next(pt.get_child("data").begin(), datafn_position)
+    ->second.get_value<std::string>();
 
   lumi.loaded = true;
   return lumi;
@@ -64,10 +58,11 @@ DQMFileIterator::EorEntry DQMFileIterator::EorEntry::load_json(
   return eor;
 }
 
-DQMFileIterator::DQMFileIterator(edm::ParameterSet const& pset, JsonType t)
-    : type_(t), state_(EOR) {
+DQMFileIterator::DQMFileIterator(edm::ParameterSet const& pset)
+    : state_(EOR) {
 
   runNumber_ = pset.getUntrackedParameter<unsigned int>("runNumber");
+  datafnPosition_ = pset.getUntrackedParameter<unsigned int>("datafnPosition");
   runInputDir_ = pset.getUntrackedParameter<std::string>("runInputDir");
   streamLabel_ = pset.getUntrackedParameter<std::string>("streamLabel");
   delayMillis_ = pset.getUntrackedParameter<uint32_t>("delayMillis");
@@ -217,7 +212,7 @@ void DQMFileIterator::collect(bool ignoreTimers) {
         continue;
       }
 
-      LumiEntry lumi_jsn = LumiEntry::load_json(fn, lumi, type_);
+      LumiEntry lumi_jsn = LumiEntry::load_json(fn, lumi, datafnPosition_);
       lumiSeen_.emplace(lumi, lumi_jsn);
       logFileAction("Found and loaded json file: ", fn);
     }
@@ -323,6 +318,9 @@ void DQMFileIterator::fillDescription(edm::ParameterSetDescription& desc) {
   desc.addUntracked<unsigned int>("runNumber")
       ->setComment("Run number passed via configuration file.");
 
+  desc.addUntracked<unsigned int>("datafnPosition", 3)
+      ->setComment("Data filename position in the positional arguments array 'data' in json file.");
+
   desc.addUntracked<std::string>("streamLabel")
       ->setComment("Stream label used in json discovery.");
 

diff --git a/DQMServices/StreamerIO/plugins/DQMFileIterator.h b/DQMServices/StreamerIO/plugins/DQMFileIterator.h
@@ -17,11 +17,6 @@ namespace dqmservices {
 
 class DQMFileIterator {
  public:
-  enum JsonType {
-    JS_PROTOBUF,
-    JS_DATA,
-  };
-
   struct LumiEntry {
     bool loaded = false;
     std::string filename;
@@ -31,7 +26,7 @@ class DQMFileIterator {
     std::string datafilename;
 
     static LumiEntry load_json(const std::string& filename, int lumiNumber,
-                               JsonType type);
+                               unsigned int datafn_position);
   };
 
   struct EorEntry {
@@ -51,7 +46,7 @@ class DQMFileIterator {
     EOR = 2,
   };
 
-  DQMFileIterator(edm::ParameterSet const& pset, JsonType t);
+  DQMFileIterator(edm::ParameterSet const& pset);
   ~DQMFileIterator();
   void initialise(int run, const std::string&, const std::string&);
 
@@ -86,14 +81,15 @@ class DQMFileIterator {
   static void fillDescription(edm::ParameterSetDescription& d);
 
  private:
-  JsonType type_;
-
   unsigned int runNumber_;
   std::string runInputDir_;
   std::string streamLabel_;
   unsigned long delayMillis_;
   long nextLumiTimeoutMillis_;
 
+  // file name position in the json file
+  unsigned int datafnPosition_;
+
   std::string runPath_;
 
   EorEntry eor_;

diff --git a/DQMServices/StreamerIO/plugins/DQMProtobufReader.cc b/DQMServices/StreamerIO/plugins/DQMProtobufReader.cc
@@ -8,7 +8,7 @@ using namespace dqmservices;
 
 DQMProtobufReader::DQMProtobufReader(edm::ParameterSet const& pset,
                                      edm::InputSourceDescription const& desc)
-    : InputSource(pset, desc), fiterator_(pset, DQMFileIterator::JS_PROTOBUF) {
+    : InputSource(pset, desc), fiterator_(pset) {
 
   flagSkipFirstLumis_ = pset.getUntrackedParameter<bool>("skipFirstLumis");
   flagEndOfRunKills_ = pset.getUntrackedParameter<bool>("endOfRunKills");
@@ -44,7 +44,10 @@ edm::InputSource::ItemType DQMProtobufReader::getNextItemType() {
     }
 
     fiterator_.delay();
-    return InputSource::IsSynchronize;
+    // BUG: for an unknown reason it fails after a certain time if we use IsSynchronize state
+    // comment out in order to block at this level
+    // the only downside is that we cannot Ctrl+C :)
+    //return InputSource::IsSynchronize;
   }
 
   // this is unreachable

diff --git a/DQMServices/StreamerIO/plugins/DQMStreamerReader.cc b/DQMServices/StreamerIO/plugins/DQMStreamerReader.cc
@@ -28,7 +28,7 @@ namespace dqmservices {
 DQMStreamerReader::DQMStreamerReader(edm::ParameterSet const& pset,
                                      edm::InputSourceDescription const& desc)
     : StreamerInputSource(pset, desc),
-      fiterator_(pset, DQMFileIterator::JS_DATA),
+      fiterator_(pset),
       streamReader_(),
       eventSkipperByID_(edm::EventSkipperByID::create(pset).release()) {
 

diff --git a/DQMServices/StreamerIO/python/DQMProtobufReader_cff.py b/DQMServices/StreamerIO/python/DQMProtobufReader_cff.py
@@ -9,6 +9,12 @@
                  VarParsing.VarParsing.varType.int,
                  "Run number.")
 
+options.register('datafnPosition',
+                 3, # default value
+                 VarParsing.VarParsing.multiplicity.singleton,
+                 VarParsing.VarParsing.varType.int,
+                 "Data filename position in the positional arguments array 'data' in json file.")
+
 options.register('runInputDir',
                  '/build1/micius/OnlineDQM_sample/', # default value
                  VarParsing.VarParsing.multiplicity.singleton,
@@ -58,6 +64,7 @@
     runNumber = cms.untracked.uint32(options.runNumber),
     runInputDir = cms.untracked.string(options.runInputDir),
     streamLabel = cms.untracked.string(options.streamLabel),
+    datafnPosition = cms.untracked.uint32(options.datafnPosition),
 
     delayMillis = cms.untracked.uint32(options.delayMillis),
     nextLumiTimeoutMillis = cms.untracked.int32(options.nextLumiTimeoutMillis),

diff --git a/DQMServices/StreamerIO/python/DQMStreamerReader_cff.py b/DQMServices/StreamerIO/python/DQMStreamerReader_cff.py
@@ -9,6 +9,12 @@
                  VarParsing.VarParsing.varType.int,
                  "Run number.")
 
+options.register('datafnPosition',
+                 3, # default value
+                 VarParsing.VarParsing.multiplicity.singleton,
+                 VarParsing.VarParsing.varType.int,
+                 "Data filename position in the positional arguments array 'data' in json file.")
+
 options.register('runInputDir',
                  '/build1/micius/OnlineDQM_sample/', # default value
                  VarParsing.VarParsing.multiplicity.singleton,
@@ -65,6 +71,7 @@
     runNumber = cms.untracked.uint32(options.runNumber),
     runInputDir = cms.untracked.string(options.runInputDir),
     streamLabel = cms.untracked.string(options.streamLabel),
+    datafnPosition = cms.untracked.uint32(options.datafnPosition),
 
     minEventsPerLumi = cms.untracked.int32(options.minEventsPerLumi),
     delayMillis = cms.untracked.uint32(options.delayMillis),