Merge pull request #19035 from dmitrijus/dt4ml

Online DQM fix for ML applications.

DQM/Integration/python/clients/dt4ml_dqm_sourceclient-live_cfg.py

@@ -0,0 +1,111 @@
+import FWCore.ParameterSet.Config as cms
+import os
+
+process = cms.Process("DTDQM")
+
+#----------------------------
+#### Event Source
+#----------------------------
+# for live online DQM in P5
+process.load("DQM.Integration.config.inputsource_cfi")
+
+# for testing in lxplus
+#process.load("DQM.Integration.config.fileinputsource_cfi")
+
+#----------------------------
+#### DQM Environment
+#----------------------------
+process.load("DQM.Integration.config.environment_cfi")
+process.DQMStore.referenceFileName = '/dqmdata/dqm/reference/dt_reference.root'
+#process.DQMStore.referenceFileName = "DT_reference.root"
+
+#----------------------------
+#### DQM Live Environment
+#----------------------------
+process.dqmEnv.subSystemFolder = 'DT'
+process.dqmSaver.tag = "DT"
+#-----------------------------
+
+### CUSTOMIZE FOR ML
+
+# prepare the output directory
+filePath = "/globalscratch/dqm4ml_" + process.dqmRunConfig.type.value()
+try:
+    os.makedirs(filePath)
+except:
+    pass
+
+process.dqmSaver.backupLumiCount = 10
+process.dqmSaver.keepBackupLumi = True
+process.dqmSaver.path = filePath
+
+# disable DQM gui
+print "old:",process.DQM.collectorHost
+process.DQM.collectorHost = cms.untracked.string('dqm-blackhole.cms')
+print "new:",process.DQM.collectorHost
+### END OF CUSTOMIZE FOR ML
+
+# DT reco and DQM sequences
+process.load("Configuration.StandardSequences.GeometryRecoDB_cff")
+process.load("Configuration/StandardSequences/MagneticField_cff")
+process.load("DQM.DTMonitorModule.dt_dqm_sourceclient_common_cff")
+#---- for P5 (online) DB access
+process.load("DQM.Integration.config.FrontierCondition_GT_cfi")
+#---- for offline DB: change and possibly customise the GT
+#from Configuration.AlCa.GlobalTag import GlobalTag as gtCustomise
+#process.GlobalTag = gtCustomise(process.GlobalTag, 'auto:run2_data', '')
+
+# message logger
+process.MessageLogger = cms.Service("MessageLogger",
+                                    destinations = cms.untracked.vstring('cout'),
+                                    cout = cms.untracked.PSet(threshold = cms.untracked.string('WARNING'))
+                                    )
+
+process.dqmmodules = cms.Sequence(process.dqmEnv + process.dqmSaver)
+
+process.dtDQMPathPhys = cms.Path(process.unpackers + process.dqmmodules + process.physicsEventsFilter *  process.dtDQMPhysSequence)
+
+#process.dtDQMPathCalib = cms.Path(process.unpackers + process.dqmmodules + process.calibrationEventsFilter * process.dtDQMCalib)
+
+process.twinMuxStage2Digis.DTTM7_FED_Source = cms.InputTag("rawDataCollector")
+process.dtunpacker.inputLabel = cms.InputTag("rawDataCollector")
+process.gtDigis.DaqGtInputTag = cms.InputTag("rawDataCollector")
+process.scalersRawToDigi.scalersInputTag = cms.InputTag("rawDataCollector")
+
+print "Running with run type = ", process.runType.getRunType()
+
+#----------------------------
+#### pp run settings 
+#----------------------------
+
+if (process.runType.getRunType() == process.runType.pp_run):
+    process.DQMStore.referenceFileName = '/dqmdata/dqm/reference/dt_reference_pp.root'
+
+
+#----------------------------
+#### cosmic run settings 
+#----------------------------
+
+if (process.runType.getRunType() == process.runType.cosmic_run):
+    process.DQMStore.referenceFileName = '/dqmdata/dqm/reference/dt_reference_cosmic.root'
+
+
+#----------------------------
+#### HI run settings 
+#----------------------------
+
+if (process.runType.getRunType() == process.runType.hi_run):
+    process.dtunpacker.fedbyType = cms.bool(False)
+    process.twinMuxStage2Digis.DTTM7_FED_Source = cms.InputTag("rawDataRepacker")
+    process.dtunpacker.inputLabel = cms.InputTag("rawDataRepacker")
+    process.gtDigis.DaqGtInputTag = cms.InputTag("rawDataRepacker")
+    process.scalersRawToDigi.scalersInputTag = cms.InputTag("rawDataRepacker")
+
+    process.dtDigiMonitor.ResetCycle = cms.untracked.int32(9999)
+
+    process.DQMStore.referenceFileName = '/dqmdata/dqm/reference/dt_reference_hi.root'
+
+
+### process customizations included here
+from DQM.Integration.config.online_customizations_cfi import *
+process = customise(process)

DQMServices/FileIO/plugins/DQMFileSaverOnline.cc

@@ -1,45 +1,44 @@
 #include "DQMServices/Core/interface/DQMStore.h"
 #include "DQMServices/Core/interface/MonitorElement.h"
 #include "FWCore/Framework/interface/Event.h"
-#include "FWCore/Framework/interface/Run.h"
 #include "FWCore/Framework/interface/LuminosityBlock.h"
+#include "FWCore/Framework/interface/Run.h"
+#include "FWCore/MessageLogger/interface/JobReport.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
-#include "FWCore/Version/interface/GetReleaseVersion.h"
 #include "FWCore/ServiceRegistry/interface/Service.h"
-#include "FWCore/MessageLogger/interface/MessageLogger.h"
-#include "FWCore/MessageLogger/interface/JobReport.h"
+#include "FWCore/Version/interface/GetReleaseVersion.h"
 
 #include "DQMFileSaverOnline.h"
 
+#include <TString.h>
+#include <TSystem.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <fstream>
 #include <iostream>
-#include <vector>
 #include <string>
-#include <fstream>
 #include <utility>
-#include <TString.h>
-#include <TSystem.h>
+#include <vector>
 
 #include <openssl/md5.h>
-#include <boost/iostreams/device/mapped_file.hpp>
 #include <boost/filesystem.hpp>
+#include <boost/iostreams/device/mapped_file.hpp>
 
 using namespace dqm;
 
-DQMFileSaverOnline::DQMFileSaverOnline(const edm::ParameterSet &ps)
+DQMFileSaverOnline::DQMFileSaverOnline(const edm::ParameterSet& ps)
     : DQMFileSaverBase(ps) {
-
   backupLumiCount_ = ps.getUntrackedParameter<int>("backupLumiCount", 1);
+  keepBackupLumi_ = ps.getUntrackedParameter<bool>("keepBackupLumi", false);
 }
 
 DQMFileSaverOnline::~DQMFileSaverOnline() {}
 
 void DQMFileSaverOnline::saveLumi(const FileParameters& fp) const {
   if (backupLumiCount_ > 0) {
     if (fp.lumi_ % backupLumiCount_ == 0) {
-
       // actual saving is done here
       makeSnapshot(fp, false);
     }
@@ -50,7 +49,8 @@ void DQMFileSaverOnline::saveRun(const FileParameters& fp) const {
   makeSnapshot(fp, true);
 }
 
-void DQMFileSaverOnline::makeSnapshot(const FileParameters& fp, bool final) const {
+void DQMFileSaverOnline::makeSnapshot(const FileParameters& fp,
+                                      bool final) const {
   int pid = getpid();
   char hostname[64];
   gethostname(hostname, 64);
@@ -75,103 +75,114 @@ void DQMFileSaverOnline::makeSnapshot(const FileParameters& fp, bool final) cons
   edm::Service<DQMStore> store;
 
   logFileAction("Writing DQM Root file: ", root_fp);
-  //logFileAction("Writing DQM Origin file: ", meta_fp);
+  // logFileAction("Writing DQM Origin file: ", meta_fp);
 
   char rewrite[128];
   snprintf(rewrite, 128, "\\1Run %ld/\\2/Run summary", fp.run_);
 
-  store->save(tmp_root_fp,                       /* filename      */
-              "",                                /* path          */
-              "^(Reference/)?([^/]+)",           /* pattern       */
-              rewrite,                           /* rewrite       */
+  store->save(tmp_root_fp,                      /* filename      */
+              "",                               /* path          */
+              "^(Reference/)?([^/]+)",          /* pattern       */
+              rewrite,                          /* rewrite       */
               store->mtEnabled() ? fp.run_ : 0, /* run           */
-              0,                                 /* lumi          */
+              0,                                /* lumi          */
               fp.saveReference_,                /* ref           */
               fp.saveReferenceQMin_,            /* ref minStatus */
-              "RECREATE",                        /* fileupdate    */
-              false                              /* resetMEs      */
+              "RECREATE",                       /* fileupdate    */
+              false                             /* resetMEs      */
               );
 
   // write metadata
-  // format.origin: md5:d566a34b27f48d507150a332b189398b 294835 /home/dqmprolocal/output/DQM_V0001_FED_R000194224.root
+  // format.origin: md5:d566a34b27f48d507150a332b189398b 294835
+  // /home/dqmprolocal/output/DQM_V0001_FED_R000194224.root
   std::ofstream meta_fd(tmp_meta_fp);
   meta_fd << fillOrigin(tmp_root_fp, root_fp);
   meta_fd.close();
 
-  checkError("Rename failed: ", root_fp, ::rename(tmp_root_fp.c_str(), root_fp.c_str()));
-  checkError("Rename failed: ", meta_fp, ::rename(tmp_meta_fp.c_str(), meta_fp.c_str()));
+  checkError("Rename failed: ", root_fp,
+             ::rename(tmp_root_fp.c_str(), root_fp.c_str()));
+  checkError("Rename failed: ", meta_fp,
+             ::rename(tmp_meta_fp.c_str(), meta_fp.c_str()));
 
-  SnapshotFiles files = { root_fp, meta_fp };
+  SnapshotFiles files = {root_fp, meta_fp};
   if (final) {
     // final will never be cleared
     appendSnapshot(SnapshotFiles{});
 
     saveJobReport(root_fp);
   } else {
-    appendSnapshot(SnapshotFiles{ root_fp, meta_fp });
+    appendSnapshot(SnapshotFiles{root_fp, meta_fp});
   }
 }
 
 void DQMFileSaverOnline::appendSnapshot(SnapshotFiles f) const {
   std::lock_guard<std::mutex> lock(snapshots_lock_);
 
-  while (! snapshots_.empty()) {
-    SnapshotFiles& x = snapshots_.front();
+  if (!keepBackupLumi_) {
+    while (!snapshots_.empty()) {
+      SnapshotFiles& x = snapshots_.front();
 
-    //logFileAction("Deleting old snapshot (origin): ", x.meta);
-    checkError("Unlink failed: ", x.meta, ::unlink(x.meta.c_str()));
+      // logFileAction("Deleting old snapshot (origin): ", x.meta);
+      checkError("Unlink failed: ", x.meta, ::unlink(x.meta.c_str()));
 
-    logFileAction("Deleting old snapshot (root): ", x.data);
-    checkError("Unlink failed: ", x.data, ::unlink(x.data.c_str()));
+      logFileAction("Deleting old snapshot (root): ", x.data);
+      checkError("Unlink failed: ", x.data, ::unlink(x.data.c_str()));
 
-    snapshots_.pop_front();
+      snapshots_.pop_front();
+    }
   }
 
-  if (! f.data.empty()) {
+  if (!f.data.empty()) {
     snapshots_.push_back(f);
   }
 }
 
-void DQMFileSaverOnline::checkError(const char *msg, const std::string file, int status) const {
+void DQMFileSaverOnline::checkError(const char* msg, const std::string file,
+                                    int status) const {
   if (status != 0) {
     std::string actual_msg = msg;
     actual_msg += std::strerror(status);
     logFileAction(actual_msg, file);
   }
 }
 
-const std::string DQMFileSaverOnline::fillOrigin(const std::string filename,
-                                  const std::string final_filename) {
+const std::string DQMFileSaverOnline::fillOrigin(
+    const std::string filename, const std::string final_filename) {
+  // format.origin (one line):
+  //   md5:d566a34b27f48d507150a332b189398b 294835 final_filename.root
 
-    // format.origin (one line):
-    //   md5:d566a34b27f48d507150a332b189398b 294835 final_filename.root
+  unsigned char md5[MD5_DIGEST_LENGTH];
 
-    unsigned char md5[MD5_DIGEST_LENGTH];
+  boost::iostreams::mapped_file_source fp(filename);
 
-    boost::iostreams::mapped_file_source fp(filename);
+  MD5((unsigned char*)fp.data(), fp.size(), md5);
 
-    MD5((unsigned char *)fp.data(), fp.size(), md5);
-
-    std::ostringstream hash;
-    for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
-      hash << std::hex << std::setfill('0') << std::setw(2) << (int)(md5[i]);
-    }
+  std::ostringstream hash;
+  for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) {
+    hash << std::hex << std::setfill('0') << std::setw(2) << (int)(md5[i]);
+  }
 
-    std::ostringstream out;
-    out << "md5:" << hash.str() << " " << fp.size() << " " << final_filename;
-    return out.str();
+  std::ostringstream out;
+  out << "md5:" << hash.str() << " " << fp.size() << " " << final_filename;
+  return out.str();
 }
 
-
-
 void DQMFileSaverOnline::fillDescriptions(
     edm::ConfigurationDescriptions& descriptions) {
-
   edm::ParameterSetDescription desc;
   desc.setComment("Saves histograms from DQM store, online workflow.");
 
-  desc.addUntracked<int>("backupLumiCount", 10)->setComment(
-      "How often the backup file will be generated, in lumisections (-1 disables).");
+  desc.addUntracked<int>("backupLumiCount", 10)
+      ->setComment(
+          "How often the backup file will be generated, in lumisections (-1 "
+          "disables).");
+
+  desc.addUntracked<bool>("keepBackupLumi", false)
+      ->setComment(
+          "Usually the backup old backup is deleted once the new file is "
+          "available. Setting this to true ensures that no backup files are "
+          "ever deleted. Useful for ML applications, which use backups as a "
+          "'history' of what happened during the run.");
 
   DQMFileSaverBase::fillDescription(desc);
 

DQMServices/FileIO/plugins/DQMFileSaverOnline.h

@@ -1,45 +1,46 @@
 #ifndef DQMSERVICES_COMPONENTS_DQMFILESAVEROUTPUT_H
 #define DQMSERVICES_COMPONENTS_DQMFILESAVEROUTPUT_H
 
-#include "FWCore/Framework/interface/global/EDAnalyzer.h"
 #include "DQMServices/Core/interface/MonitorElement.h"
+#include "FWCore/Framework/interface/global/EDAnalyzer.h"
 
 #include <sys/time.h>
-#include <string>
 #include <mutex>
+#include <string>
 
 #include "DQMFileSaverBase.h"
 
 namespace dqm {
 
 class DQMFileSaverOnline : public DQMFileSaverBase {
  public:
-  DQMFileSaverOnline(const edm::ParameterSet &ps);
+  DQMFileSaverOnline(const edm::ParameterSet& ps);
   ~DQMFileSaverOnline();
 
   static const std::string fillOrigin(const std::string filename,
-                                  const std::string final_filename);
+                                      const std::string final_filename);
 
  protected:
   virtual void saveLumi(const FileParameters& fp) const override;
   virtual void saveRun(const FileParameters& fp) const override;
 
  protected:
   int backupLumiCount_;
+  bool keepBackupLumi_;
 
   // snapshot making
   struct SnapshotFiles {
     std::string data;
     std::string meta;
   };
 
-  void makeSnapshot(const FileParameters &fp, bool final) const;
+  void makeSnapshot(const FileParameters& fp, bool final) const;
   void appendSnapshot(SnapshotFiles new_snap) const;
 
   mutable std::mutex snapshots_lock_;
   mutable std::list<SnapshotFiles> snapshots_;
 
-  void checkError(const char *msg, const std::string file, int status) const;
+  void checkError(const char* msg, const std::string file, int status) const;
 
  public:
   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);

DQMServices/FileIO/python/DQMFileSaverOnline_cfi.py

@@ -17,4 +17,7 @@
 
     # How often the backup file will be generated, in lumisections (-1 disables).
     backupLumiCount = cms.untracked.int32(-1),
+
+    # Set to true to preserve 'lumi backup'.
+    keepBackupLumi = cms.untracked.bool(False)
 )