Merge pull request #6637 from dmitrijus/eventdisplay_json_hax2

Add JsonWritingTimeoutPoolOutputModule.

DQMServices/Components/src/DQMFileSaver.cc

@@ -237,16 +237,19 @@ DQMFileSaver::saveForOnline(int run, const std::string &suffix, const std::strin
 }
 
 
-void
-DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, boost::property_tree::ptree& pt) const
+boost::property_tree::ptree
+DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, evf::FastMonitoringService *fms)
 {
   namespace bpt = boost::property_tree;
   namespace bfs = boost::filesystem;
+
+  bpt::ptree pt;
+
   int hostnameReturn;
   char host[32];
   hostnameReturn = gethostname(host ,sizeof(host));
   if (hostnameReturn == -1)
-    throw cms::Exception("DQMFileSaver")
+    throw cms::Exception("fillJson")
           << "Internal error, cannot get host name";
 
   int pid = getpid();
@@ -256,7 +259,7 @@ DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, b
   // Stat the data file: if not there, throw
   struct stat dataFileStat;
   if (stat(dataFilePathName.c_str(), &dataFileStat) != 0)
-    throw cms::Exception("DQMFileSaver")
+    throw cms::Exception("fillJson")
           << "Internal error, cannot get data file: "
           << dataFilePathName;
   // Extract only the data file name from the full path
@@ -265,8 +268,8 @@ DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, b
   bpt::ptree data;
   bpt::ptree processedEvents, acceptedEvents, errorEvents, bitmask, fileList, fileSize, inputFiles, fileAdler32;
 
-  processedEvents.put("", fms_ ? (fms_->getEventsProcessedForLumi(lumi)) : -1); // Processed events
-  acceptedEvents.put("", fms_ ? (fms_->getEventsProcessedForLumi(lumi)) : -1); // Accepted events, same as processed for our purposes
+  processedEvents.put("", fms ? (fms->getEventsProcessedForLumi(lumi)) : -1); // Processed events
+  acceptedEvents.put("", fms ? (fms->getEventsProcessedForLumi(lumi)) : -1); // Accepted events, same as processed for our purposes
 
   errorEvents.put("", 0); // Error events
   bitmask.put("", 0); // Bitmask of abs of CMSSW return code
@@ -286,7 +289,7 @@ DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, b
 
   pt.add_child("data", data);
 
-  if (fakeFilterUnitMode_) {
+  if (fms == nullptr) {
     pt.put("definition", "/fakeDefinition.jsn");
   } else {
     // The availability test of the EvFDaqDirector Service was done in the ctor.
@@ -298,14 +301,15 @@ DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, b
   char sourceInfo[64]; //host and pid information
   sprintf(sourceInfo, "%s_%d", host, pid);
   pt.put("source", sourceInfo);
+
+  return pt;
 }
 
 void
 DQMFileSaver::saveForFilterUnit(const std::string& rewrite, int run, int lumi,  const DQMFileSaver::FileFormat fileFormat) const
 {
   // get from DAQ2 services where to store the files according to their format
   namespace bpt = boost::property_tree;
-  bpt::ptree pt;
 
   std::string openJsonFilePathName;
   std::string jsonFilePathName;
@@ -372,7 +376,7 @@ DQMFileSaver::saveForFilterUnit(const std::string& rewrite, int run, int lumi,
   rename(openHistoFilePathName.c_str(), histoFilePathName.c_str());
 
   // Write the json file in the open directory.
-  fillJson(run, lumi, histoFilePathName, pt);
+  bpt::ptree pt = fillJson(run, lumi, histoFilePathName, fms_);
   write_json(openJsonFilePathName, pt);
   rename(openJsonFilePathName.c_str(), jsonFilePathName.c_str());
 }

DQMServices/Components/src/DQMFileSaver.h

@@ -16,6 +16,14 @@ class DQMFileSaver : public edm::global::EDAnalyzer<edm::RunCache<saverDetails::
 {
 public:
   DQMFileSaver(const edm::ParameterSet &ps);
+
+
+  // used by the JsonWritingTimedPoolOutputModule,
+  // fms will be nullptr in such case
+  static boost::property_tree::ptree fillJson(
+      int run, int lumi, const std::string &dataFilePathName,
+      evf::FastMonitoringService *fms);
+
 
 protected:
   virtual void beginJob(void);
@@ -50,7 +58,6 @@ class DQMFileSaver : public edm::global::EDAnalyzer<edm::RunCache<saverDetails::
 
   void saveForFilterUnit(const std::string& rewrite, int run, int lumi, const FileFormat fileFormat) const;
   void saveJobReport(const std::string &filename) const;
-  void fillJson(int run, int lumi, const std::string& dataFilePathName, boost::property_tree::ptree& pt) const;
 
   Convention	convention_;
   FileFormat    fileFormat_;

DQMServices/StreamerIO/plugins/BuildFile.xml

@@ -1,8 +1,10 @@
 <use   name="FWCore/Framework"/>
 <use   name="FWCore/ServiceRegistry"/>
 <use   name="IOPool/Streamer"/>
+<use   name="IOPool/Output"/>
 <use   name="EventFilter/Utilities"/>
 <use   name="DQMServices/Core"/>
+<use   name="DQMServices/Components"/>
 
 <library   file="*.cc" name="DQMServicesStreamerIOPlugins">
   <flags   EDM_PLUGIN="1"/>

DQMServices/StreamerIO/plugins/JsonWritingTimeoutPoolOutputModule.cc

@@ -0,0 +1,73 @@
+#include "JsonWritingTimeoutPoolOutputModule.h"
+
+#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h"
+#include "DQMServices/Components/src/DQMFileSaver.h"
+
+#include <boost/format.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/property_tree/json_parser.hpp>
+#include <boost/property_tree/ptree.hpp>
+
+namespace dqmservices {
+
+JsonWritingTimeoutPoolOutputModule::JsonWritingTimeoutPoolOutputModule(
+    edm::ParameterSet const& ps)
+    : edm::one::OutputModuleBase::OutputModuleBase(ps),
+      edm::TimeoutPoolOutputModule(ps) {
+  runNumber_ = ps.getUntrackedParameter<uint32_t>("runNumber");
+  outputPath_ = ps.getUntrackedParameter<std::string>("outputPath");
+  streamLabel_ = ps.getUntrackedParameter<std::string>("streamLabel");
+
+  sequence_ = 0;
+}
+
+std::pair<std::string, std::string>
+JsonWritingTimeoutPoolOutputModule::physicalAndLogicalNameForNewFile() {
+  sequence_++;
+
+  std::string base = str(boost::format("run%06d_ls%04d_%s") % runNumber_ %
+                         sequence_ % streamLabel_);
+
+  boost::filesystem::path p(outputPath_);
+
+  currentFileName_ = (p / base).string() + ".root";
+  currentJsonName_ = (p / base).string() + ".jsn";
+
+  return std::make_pair(currentFileName_, currentFileName_);
+}
+
+void JsonWritingTimeoutPoolOutputModule::doExtrasAfterCloseFile() {
+  std::string json_tmp_ = currentJsonName_ + ".open";
+  auto pt =
+      DQMFileSaver::fillJson(runNumber_, sequence_, currentFileName_, nullptr);
+  write_json(json_tmp_, pt);
+  rename(json_tmp_.c_str(), currentJsonName_.c_str());
+}
+
+void JsonWritingTimeoutPoolOutputModule::fillDescriptions(
+    edm::ConfigurationDescriptions& descriptions) {
+  edm::ParameterSetDescription desc;
+  TimeoutPoolOutputModule::fillDescription(desc);
+
+  desc.setComment(
+      "Almost same as TimeoutPoolOutputModule, but the output files names "
+      "follow the FFF naming convention. Additionally a json 'description' "
+      "file is emitted for every .root file written.");
+
+  desc.addUntracked<uint32_t>("runNumber", 0)->setComment(
+      "The run number, only used for file prefix: 'run000001_lumi0000_...'.");
+
+  desc.addUntracked<std::string>("outputPath", "./")->setComment(
+      "Output path for the root and json files, usually the run directory.");
+
+  desc.addUntracked<std::string>("streamLabel", "streamEvDOutput")
+      ->setComment("Stream label, used for file suffix.");
+
+  descriptions.add("jsonWriting", desc);
+}
+
+}  // end of namespace
+
+#include "FWCore/Framework/interface/MakerMacros.h"
+using dqmservices::JsonWritingTimeoutPoolOutputModule;
+DEFINE_FWK_MODULE(JsonWritingTimeoutPoolOutputModule);

DQMServices/StreamerIO/plugins/JsonWritingTimeoutPoolOutputModule.h

@@ -0,0 +1,35 @@
+#ifndef DQMServices_StreamerIO_JsonWritingTimeoutPoolOutputModule_h
+#define DQMServices_StreamerIO_JsonWritingTimeoutPoolOutputModule_h
+
+#include "IOPool/Output/interface/TimeoutPoolOutputModule.h"
+
+namespace dqmservices {
+
+class ModuleCallingContext;
+class ParameterSet;
+
+class JsonWritingTimeoutPoolOutputModule : public edm::TimeoutPoolOutputModule {
+ public:
+  explicit JsonWritingTimeoutPoolOutputModule(edm::ParameterSet const& ps);
+  virtual ~JsonWritingTimeoutPoolOutputModule(){};
+
+  static void fillDescriptions(edm::ConfigurationDescriptions&);
+
+ protected:
+  virtual std::pair<std::string, std::string> physicalAndLogicalNameForNewFile()
+      override;
+  virtual void doExtrasAfterCloseFile() override;
+
+ protected:
+  uint32_t sequence_;
+  uint32_t runNumber_;
+  std::string streamLabel_;
+  std::string outputPath_;
+
+  std::string currentFileName_;
+  std::string currentJsonName_;
+};
+
+}  // end of namespace
+
+#endif

DQMServices/StreamerIO/test/DQMStreamerOutputModule.cc

@@ -6,8 +6,8 @@
 
 #include <sstream>
 #include <iomanip>
-#include "boost/filesystem.hpp"
-#include "boost/format.hpp"
+#include <boost/filesystem.hpp>
+#include <boost/format.hpp>
 #include <boost/property_tree/json_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 

IOPool/Output/interface/PoolOutputModule.h

@@ -52,6 +52,7 @@ namespace edm {
 
     std::string const& currentFileName() const;
 
+    static void fillDescription(ParameterSetDescription& desc);
     static void fillDescriptions(ConfigurationDescriptions& descriptions);
 
     using OutputModule::selectorConfig;
@@ -102,6 +103,9 @@ namespace edm {
     ///allow inheriting classes to override but still be able to call this method in the overridden version
     virtual bool shouldWeCloseFile() const override;
     virtual void write(EventPrincipal const& e, ModuleCallingContext const*) override;
+
+    virtual std::pair<std::string, std::string> physicalAndLogicalNameForNewFile();
+    virtual void doExtrasAfterCloseFile();
   private:
     virtual void openFile(FileBlock const& fb) override;
     virtual void respondToOpenInputFile(FileBlock const& fb) override;

IOPool/Output/src/PoolOutputModule.cc

@@ -268,13 +268,14 @@ namespace edm {
     writeBranchIDListRegistry();
     writeProductDependencies();
     finishEndFile();
+
+    doExtrasAfterCloseFile();
   }
 
 
   // At some later date, we may move functionality from finishEndFile() to here.
   void PoolOutputModule::startEndFile() { }
 
-
   void PoolOutputModule::writeFileFormatVersion() { rootOutputFile_->writeFileFormatVersion(); }
   void PoolOutputModule::writeFileIdentifier() { rootOutputFile_->writeFileIdentifier(); }
   void PoolOutputModule::writeIndexIntoFile() { rootOutputFile_->writeIndexIntoFile(); }
@@ -285,10 +286,12 @@ namespace edm {
   void PoolOutputModule::writeBranchIDListRegistry() { rootOutputFile_->writeBranchIDListRegistry(); }
   void PoolOutputModule::writeProductDependencies() { rootOutputFile_->writeProductDependencies(); }
   void PoolOutputModule::finishEndFile() { rootOutputFile_->finishEndFile(); rootOutputFile_.reset(); }
+  void PoolOutputModule::doExtrasAfterCloseFile() {}
   bool PoolOutputModule::isFileOpen() const { return rootOutputFile_.get() != 0; }
   bool PoolOutputModule::shouldWeCloseFile() const { return rootOutputFile_->shouldWeCloseFile(); }
 
-  void PoolOutputModule::reallyOpenFile() {
+  std::pair<std::string, std::string>
+  PoolOutputModule::physicalAndLogicalNameForNewFile() {
       if(inputFileCount_ == 0) {
         throw edm::Exception(errors::LogicError)
           << "Attempt to open output file before input file. "
@@ -316,14 +319,20 @@ namespace edm {
         }
       }
       ofilename << suffix;
-      rootOutputFile_.reset(new RootOutputFile(this, ofilename.str(), lfilename.str()));
       ++outputFileCount_;
+
+      return std::make_pair(ofilename.str(), lfilename.str());
+  }
+
+  void PoolOutputModule::reallyOpenFile() {
+    auto names = physicalAndLogicalNameForNewFile();
+    rootOutputFile_.reset( new RootOutputFile(this, names.first, names.second));
   }
 
   void
-  PoolOutputModule::fillDescriptions(ConfigurationDescriptions & descriptions) {
+  PoolOutputModule::fillDescription(ParameterSetDescription& desc) {
     std::string defaultString;
-    ParameterSetDescription desc;
+
     desc.setComment("Writes runs, lumis, and events into EDM/ROOT files.");
     desc.addUntracked<std::string>("fileName")
         ->setComment("Name of output file.");
@@ -370,7 +379,12 @@ namespace edm {
      ->setComment("PSet is only used by Data Operations and not by this module.");
 
     OutputModule::fillDescription(desc);
+  }
 
+  void
+  PoolOutputModule::fillDescriptions(ConfigurationDescriptions & descriptions) {
+    ParameterSetDescription desc;
+    PoolOutputModule::fillDescription(desc);
     descriptions.add("edmOutput", desc);
   }
 }