Merge pull request #14722 from ghellwig/fix-memory-blowup-ALCAHARVEST…

…_80X

Fix memory blowup in ALCAHARVEST 80X (backport of #14721)

Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli0T_cff.py

@@ -99,6 +99,7 @@
 
 SiPixelAliMilleAlignmentProducer.algoConfig = MillePedeAlignmentAlgorithm
 SiPixelAliMilleAlignmentProducer.algoConfig.mode = 'mille'
+SiPixelAliMilleAlignmentProducer.algoConfig.runAtPCL = True
 SiPixelAliMilleAlignmentProducer.algoConfig.mergeBinaryFiles = cms.vstring()
 SiPixelAliMilleAlignmentProducer.algoConfig.binaryFile = 'milleBinary_0.dat'
 SiPixelAliMilleAlignmentProducer.algoConfig.TrajectoryFactory = cms.PSet(

Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli_cff.py

@@ -97,6 +97,7 @@
 
 SiPixelAliMilleAlignmentProducer.algoConfig = MillePedeAlignmentAlgorithm
 SiPixelAliMilleAlignmentProducer.algoConfig.mode = 'mille'
+SiPixelAliMilleAlignmentProducer.algoConfig.runAtPCL = True
 SiPixelAliMilleAlignmentProducer.algoConfig.mergeBinaryFiles = cms.vstring()
 SiPixelAliMilleAlignmentProducer.algoConfig.binaryFile = 'milleBinary_0.dat'
 SiPixelAliMilleAlignmentProducer.algoConfig.TrajectoryFactory = cms.PSet(

Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeAlignmentAlgorithm.h

@@ -90,6 +90,12 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase
   // This one will be called since it matches the interface of the base class
   virtual void endRun(const EndRunInfo &runInfo, const edm::EventSetup &setup);
 
+  /// called at begin of luminosity block (resets Mille binary in mille mode)
+  virtual void beginLuminosityBlock(const edm::EventSetup&) override;
+
+  /// called at end of luminosity block
+  virtual void endLuminosityBlock(const edm::EventSetup&) override;
+
 
 /*   virtual void beginLuminosityBlock(const edm::EventSetup &setup) {} */
 /*   virtual void endLuminosityBlock(const edm::EventSetup &setup) {} */
@@ -254,6 +260,8 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase
   // CHK for GBL
   std::unique_ptr<gbl::MilleBinary> theBinary;
   bool                      theGblDoubleBinary;
+
+  const bool                runAtPCL_;
 };
 
 #endif

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.cc

@@ -10,42 +10,41 @@
 #include <fstream>
 
 MillePedeFileConverter::MillePedeFileConverter(const edm::ParameterSet& iConfig)
-    : theInputDir(iConfig.getParameter<std::string>("fileDir")),
-      theInputFileName(iConfig.getParameter<std::string>("inputBinaryFile")),
-      theFileBlobLabel(iConfig.getParameter<std::string>("fileBlobLabel")) {
+    : inputDir_(iConfig.getParameter<std::string>("fileDir")),
+      inputFileName_(iConfig.getParameter<std::string>("inputBinaryFile")),
+      fileBlobLabel_(iConfig.getParameter<std::string>("fileBlobLabel")) {
   // We define what this producer produces: A FileBlobCollection
-  produces<FileBlobCollection, edm::InRun>(theFileBlobLabel);
+  produces<FileBlobCollection, edm::InLumi>(fileBlobLabel_);
 }
 
 MillePedeFileConverter::~MillePedeFileConverter() {}
 
-void MillePedeFileConverter::endRunProduce(edm::Run& iRun,
-                                           const edm::EventSetup& iSetup) {
+void MillePedeFileConverter::endLuminosityBlockProduce(edm::LuminosityBlock& iLumi,
+                                                       const edm::EventSetup& iSetup) {
   edm::LogInfo("MillePedeFileActions")
-      << "Inserting all data from file " << theInputDir + theInputFileName
-      << " as a FileBlob to the run, using label \"" << theFileBlobLabel
+      << "Inserting all data from file " << inputDir_ + inputFileName_
+      << " as a FileBlob to the lumi, using label \"" << fileBlobLabel_
       << "\".";
   // Preparing the FileBlobCollection:
-  std::unique_ptr<FileBlobCollection> theFileBlobCollection(
-      new FileBlobCollection());
-  FileBlob theFileBlob;
+  auto fileBlobCollection = std::make_unique<FileBlobCollection>();
+  FileBlob fileBlob;
   try {
     // Creating the FileBlob:
     // (The FileBlob will signal problems with the file itself.)
-    theFileBlob = FileBlob(theInputDir + theInputFileName, true);
+    fileBlob = FileBlob(inputDir_ + inputFileName_, true);
   }
   catch (...) {
     // When creation of the FileBlob fails:
     edm::LogError("MillePedeFileActions")
         << "Error: No FileBlob could be created from the file \""
-        << theInputDir + theInputFileName << "\".";
+        << inputDir_ + inputFileName_ << "\".";
     throw;
   }
-  if (theFileBlob.size() > 0) {
-    // Adding the FileBlob to the run:
-    theFileBlobCollection->addFileBlob(theFileBlob);
-    iRun.put(std::move(theFileBlobCollection), theFileBlobLabel);
+  if (fileBlob.size() > 0) {
+    // Adding the FileBlob to the lumi:
+    fileBlobCollection->addFileBlob(fileBlob);
   }
+  iLumi.put(std::move(fileBlobCollection), fileBlobLabel_);
 }
 
 // Manage the parameters for the module:

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileConverter.h

@@ -26,20 +26,20 @@
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
 
 class MillePedeFileConverter
-    : public edm::one::EDProducer<edm::EndRunProducer> {
+    : public edm::one::EDProducer<edm::EndLuminosityBlockProducer> {
  public:
   explicit MillePedeFileConverter(const edm::ParameterSet&);
   ~MillePedeFileConverter();
   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
 
  private:
   virtual void produce(edm::Event&, const edm::EventSetup&) override {}
-  virtual void endRunProduce(edm::Run& run,
-                             const edm::EventSetup& iSetup) override final;
+  virtual void endLuminosityBlockProduce(edm::LuminosityBlock&,
+                                         const edm::EventSetup&) override final;
 
-  std::string theInputDir;
-  std::string theInputFileName;
-  std::string theFileBlobLabel;
+  const std::string inputDir_;
+  const std::string inputFileName_;
+  const std::string fileBlobLabel_;
 };
 
 // define this as a plug-in

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc

@@ -1,62 +1,67 @@
 // Original Author:  Broen van Besien
 //         Created:  Mon, 23 Mar 2015 14:56:15 GMT
 
-#include <memory>
-
 #include "Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h"
 
 #include "FWCore/MessageLogger/interface/MessageLogger.h"
 #include "CondFormats/Common/interface/FileBlob.h"
 #include "FWCore/Utilities/interface/InputTag.h"
-#include "FWCore/Utilities/interface/EDGetToken.h" 
+#include "FWCore/Utilities/interface/EDGetToken.h"
 
 MillePedeFileExtractor::MillePedeFileExtractor(const edm::ParameterSet& iConfig)
-    : theOutputDir(iConfig.getParameter<std::string>("fileDir")),
-      theOutputFileName(iConfig.getParameter<std::string>("outputBinaryFile")) {
+    : outputDir_(iConfig.getParameter<std::string>("fileDir")),
+      outputFileName_(iConfig.getParameter<std::string>("outputBinaryFile")),
+      maxNumberOfBinaries_(iConfig.getParameter<int>("maxNumberOfBinaries")) {
 
-  edm::InputTag fileBlobInputTag = iConfig.getParameter<edm::InputTag>("fileBlobInputTag");
-  theFileBlobToken = consumes<FileBlobCollection, edm::BranchType::InRun>(fileBlobInputTag);
-  // nothing else in the constructor
+  auto fileBlobInputTag = iConfig.getParameter<edm::InputTag>("fileBlobInputTag");
+  fileBlobToken_ = consumes<FileBlobCollection, edm::BranchType::InLumi>(fileBlobInputTag);
+  if (hasBinaryNumberLimit()) {
+    edm::LogInfo("MillePedeFileActions")
+      << "Limiting the number of extracted binary files to "
+      << maxNumberOfBinaries_;
+  }
 }
 
 MillePedeFileExtractor::~MillePedeFileExtractor() {}
 
-void MillePedeFileExtractor::endRun(const edm::Run& iRun,
-                                    edm::EventSetup const&) {
+void MillePedeFileExtractor::endLuminosityBlock(const edm::LuminosityBlock& iLumi,
+                                                const edm::EventSetup&)
+{
+  if (enoughBinaries()) return;
+
   // Getting our hands on the vector of FileBlobs
-  edm::Handle<FileBlobCollection> theFileBlobCollection;
-  iRun.getByToken(theFileBlobToken, theFileBlobCollection);
-  if (theFileBlobCollection.isValid()) {
+  edm::Handle<FileBlobCollection> fileBlobCollection;
+  iLumi.getByToken(fileBlobToken_, fileBlobCollection);
+  if (fileBlobCollection.isValid()) {
     // Logging the amount of FileBlobs in the vector
-    int theVectorSize = theFileBlobCollection->size();
-    edm::LogInfo("MillePedeFileActions") << "Root file contains "
-                                         << theVectorSize << " FileBlob(s).";
+    edm::LogInfo("MillePedeFileActions")
+      << "Root file contains " << fileBlobCollection->size() << " FileBlob(s).";
     // Loop over the FileBlobs in the vector, and write them to files:
-    for (std::vector<FileBlob>::const_iterator it =
-             theFileBlobCollection->begin();
-         it != theFileBlobCollection->end(); ++it) {
+    for (const auto& blob: *fileBlobCollection) {
+      if (enoughBinaries()) break;
       // We format the filename with a number, starting from 0 to the size of
       // our vector.
       // For this to work, the outputBinaryFile config parameter must contain a
       // formatting directive for a number, like %04d.
       char theNumberedOutputFileName[200];
-      int theNumber = it - theFileBlobCollection->begin();
-      sprintf(theNumberedOutputFileName, theOutputFileName.c_str(), theNumber);
+      sprintf(theNumberedOutputFileName, outputFileName_.c_str(), nBinaries_);
       // Log the filename to which we will write...
       edm::LogInfo("MillePedeFileActions")
           << "Writing FileBlob file to file "
-          << theOutputDir + theNumberedOutputFileName << ".";
+          << outputDir_ + theNumberedOutputFileName << ".";
       // ...and perform the writing operation.
-      it->write(theOutputDir + theNumberedOutputFileName);
-      // Carefull, it seems that when writing to an impossible file, this is
-      // swallowed by the FileBlob->write operation and no error is thrown.
+      blob.write(outputDir_ + theNumberedOutputFileName);
+      // Careful, it seems that when writing to an impossible file, this is
+      // swallowed by the FileBlob.write operation and no error is thrown.
+      ++nBinaries_;
     }
   } else {
     edm::LogError("MillePedeFileActions")
         << "Error: The root file does not contain any vector of FileBlob.";
   }
 }
 
+
 // Manage the parameters for the module:
 // (Note that this will autogenerate the _cfi.py file.)
 void MillePedeFileExtractor::fillDescriptions(
@@ -78,6 +83,9 @@ void MillePedeFileExtractor::fillDescriptions(
       "root file. Make sure you overwrite this, if you have changed "
       "this is the configuration of the MillePedeFileConverter.");
 
+  desc.add<int>("maxNumberOfBinaries", 1000)->setComment(
+      "Number of binaries to be extracted from the input files. "
+      "Use a negative value to apply no limit.");
 
   descriptions.add("millePedeFileExtractor", desc);
   descriptions.setComment(

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h

@@ -22,27 +22,37 @@
  */
 
 #include "FWCore/Framework/interface/Frameworkfwd.h"
-#include "FWCore/Framework/interface/EDAnalyzer.h"
+#include "FWCore/Framework/interface/one/EDAnalyzer.h"
 #include "FWCore/Framework/interface/Event.h"
 #include "FWCore/Framework/interface/MakerMacros.h"
 #include "FWCore/ParameterSet/interface/ParameterSet.h"
 #include "CondFormats/Common/interface/FileBlobCollection.h"
 
-class MillePedeFileExtractor : public edm::EDAnalyzer {
+class MillePedeFileExtractor :
+  public edm::one::EDAnalyzer<edm::one::WatchLuminosityBlocks> {
  public:
   explicit MillePedeFileExtractor(const edm::ParameterSet&);
   ~MillePedeFileExtractor();
   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);
 
  private:
-  virtual void endRun(edm::Run const&, edm::EventSetup const&) override;
-  void analyze(const edm::Event&, const edm::EventSetup&) {}
+  virtual void beginLuminosityBlock(const edm::LuminosityBlock&,
+                                    const edm::EventSetup&) override {}
+  virtual void endLuminosityBlock(const edm::LuminosityBlock&,
+                                  const edm::EventSetup&) override;
+  virtual void analyze(const edm::Event&, const edm::EventSetup&) override {}
 
-  std::string theOutputDir;
-  std::string theOutputFileName;
+  bool enoughBinaries() {
+    return (nBinaries_ >= maxNumberOfBinaries_) && hasBinaryNumberLimit(); }
+  bool hasBinaryNumberLimit() { return maxNumberOfBinaries_ > -1; }
 
-  edm::EDGetTokenT<FileBlobCollection> theFileBlobToken;
+  const std::string outputDir_;
+  const std::string outputFileName_;
 
+  edm::EDGetTokenT<FileBlobCollection> fileBlobToken_;
+
+  const int maxNumberOfBinaries_;
+  int nBinaries_{0};
 };
 
 // define this as a plug-in

Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py

@@ -21,6 +21,8 @@
 
     monitorFile = cms.untracked.string('millePedeMonitor.root'), ## if empty: no monitoring...
 
+    runAtPCL = cms.bool(False), # at the PCL the mille binaries are reset at lumi-section boundaries
+
     # PSet that allows to configure the pede labeler, i.e. select the actual
     # labeler plugin to use and parameters for the selected plugin
     pedeLabeler = cms.PSet(