Merge pull request #14961 from ghellwig/reduce-PCL-disk-space_81X

Reduce PCL disk space 81X

Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli0T_cff.py

@@ -83,10 +83,10 @@
         'TrackerTPBHalfBarrel,111111',
         'TrackerTPEHalfCylinder,111111',
 
-        'TrackerTIBHalfBarrel,ffffff',
-        'TrackerTOBHalfBarrel,ffffff',
-        'TrackerTIDEndcap,ffffff',
-        'TrackerTECEndcap,ffffff'
+        'TrackerTIBHalfBarrel,000000',
+        'TrackerTOBHalfBarrel,000000',
+        'TrackerTIDEndcap,000000',
+        'TrackerTECEndcap,000000'
         )
     )
 

Alignment/CommonAlignmentProducer/python/ALCARECOPromptCalibProdSiPixelAli_cff.py

@@ -81,10 +81,10 @@
         'TrackerTPBHalfBarrel,111111',
         'TrackerTPEHalfCylinder,111111',
 
-        'TrackerTIBHalfBarrel,ffffff',
-        'TrackerTOBHalfBarrel,ffffff',
-        'TrackerTIDEndcap,ffffff',
-        'TrackerTECEndcap,ffffff'
+        'TrackerTIBHalfBarrel,000000',
+        'TrackerTOBHalfBarrel,000000',
+        'TrackerTIDEndcap,000000',
+        'TrackerTECEndcap,000000'
         )
     )
 

Alignment/CommonAlignmentProducer/python/AlcaSiPixelAliHarvester0T_cff.py

@@ -18,10 +18,10 @@
         'TrackerTPBHalfBarrel,111111',
         'TrackerTPEHalfCylinder,111111',
 
-        'TrackerTIBHalfBarrel,ffffff',
-        'TrackerTOBHalfBarrel,ffffff',
-        'TrackerTIDEndcap,ffffff',
-        'TrackerTECEndcap,ffffff'
+        'TrackerTIBHalfBarrel,000000',
+        'TrackerTOBHalfBarrel,000000',
+        'TrackerTIDEndcap,000000',
+        'TrackerTECEndcap,000000'
         )
     )
 

Alignment/CommonAlignmentProducer/python/AlcaSiPixelAliHarvester_cff.py

@@ -21,10 +21,10 @@
         'TrackerTPBHalfBarrel,111111',
         'TrackerTPEHalfCylinder,111111',
 
-        'TrackerTIBHalfBarrel,ffffff',
-        'TrackerTOBHalfBarrel,ffffff',
-        'TrackerTIDEndcap,ffffff',
-        'TrackerTECEndcap,ffffff'
+        'TrackerTIBHalfBarrel,000000',
+        'TrackerTOBHalfBarrel,000000',
+        'TrackerTIDEndcap,000000',
+        'TrackerTECEndcap,000000'
         )
     )
 

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.cc

@@ -9,24 +9,21 @@
  */
 
 #include "MillePedeAlignmentAlgorithm.h"
-//#include "MillePedeAlignmentAlgorithm.h"
 
 #include "FWCore/Framework/interface/ESHandle.h"
 #include "FWCore/Framework/interface/EventSetup.h"
 #include "FWCore/MessageLogger/interface/MessageLogger.h"
 
 #include "TrackingTools/PatternTools/interface/Trajectory.h"
-// in header, too
-// end in header, too
 
 #include "Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeMonitor.h"
 #include "Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeVariables.h"
 #include "Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeVariablesIORoot.h"
 #include "Alignment/MillePedeAlignmentAlgorithm/src/Mille.h"       // 'unpublished' interface located in src
 #include "Alignment/MillePedeAlignmentAlgorithm/src/PedeSteerer.h" // dito
 #include "Alignment/MillePedeAlignmentAlgorithm/src/PedeReader.h" // dito
-#include "Alignment/MillePedeAlignmentAlgorithm/interface/PedeLabelerBase.h" // dito
-#include "Alignment/MillePedeAlignmentAlgorithm/interface/PedeLabelerPluginFactory.h" // dito
+#include "Alignment/MillePedeAlignmentAlgorithm/interface/PedeLabelerBase.h"
+#include "Alignment/MillePedeAlignmentAlgorithm/interface/PedeLabelerPluginFactory.h"
 
 #include "Alignment/ReferenceTrajectories/interface/TrajectoryFactoryBase.h"
 #include "Alignment/ReferenceTrajectories/interface/TrajectoryFactoryPlugin.h"
@@ -96,7 +93,8 @@ MillePedeAlignmentAlgorithm::MillePedeAlignmentAlgorithm(const edm::ParameterSet
   theMaximalCor2D(cfg.getParameter<double>("max2Dcorrelation")),
   theLastWrittenIov(0),
   theGblDoubleBinary(cfg.getParameter<bool>("doubleBinary")),
-  runAtPCL_(cfg.getParameter<bool>("runAtPCL"))
+  runAtPCL_(cfg.getParameter<bool>("runAtPCL")),
+  ignoreHitsWithoutGlobalDerivatives_(cfg.getParameter<bool>("ignoreHitsWithoutGlobalDerivatives"))
 {
   if (!theDir.empty() && theDir.find_last_of('/') != theDir.size()-1) theDir += '/';// may need '/'
   edm::LogInfo("Alignment") << "@SUB=MillePedeAlignmentAlgorithm" << "Start in mode '"
@@ -578,9 +576,11 @@ int MillePedeAlignmentAlgorithm::addMeasurementData(const edm::EventSetup &setup
 					tsos, alidet, alidet, theFloatBufferX, // 2x alidet, sic!
 					theFloatBufferY, theIntBuffer, params)) {
     return -1; // problem
-  } else if (theFloatBufferX.empty()) {
+  } else if (theFloatBufferX.empty() && ignoreHitsWithoutGlobalDerivatives_) {
      return 0; // empty for X: no alignable for hit, nor calibrations
-  } else { // now even if no alignable, but calibrations!
+  } else {
+    // store measurement even if no alignable or calibrations
+    // -> measurement used for pede-internal track-fit
     return this->callMille(refTrajPtr, iHit, theIntBuffer, theFloatBufferX, theFloatBufferY);
   }
 }
@@ -1086,7 +1086,10 @@ void MillePedeAlignmentAlgorithm::diagonalize
 
   //edm::LogInfo("Alignment") << "NEW HIT loca in matrix after diag:"<<aLocalDerivativesM(0,0);
   aHitResidualsM      = aTranfoToDiagonalSystemInvF * aHitResidualsM;
-  aGlobalDerivativesM = aTranfoToDiagonalSystemInvF * aGlobalDerivativesM;
+  if (aGlobalDerivativesM.GetNoElements() > 0) {
+    // diagnoalize only if measurement depends on alignables or calibrations
+    aGlobalDerivativesM = aTranfoToDiagonalSystemInvF * aGlobalDerivativesM;
+  }
 }
 
 //__________________________________________________________________________________________________

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.h

@@ -263,6 +263,7 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase
   bool                      theGblDoubleBinary;
 
   const bool                runAtPCL_;
+  const bool                ignoreHitsWithoutGlobalDerivatives_;
 };
 
 DEFINE_EDM_PLUGIN(AlignmentAlgorithmPluginFactory,

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.cc

@@ -3,6 +3,8 @@
 
 #include "Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h"
 
+#include <zlib.h>
+
 #include "FWCore/MessageLogger/interface/MessageLogger.h"
 #include "CondFormats/Common/interface/FileBlob.h"
 #include "FWCore/Utilities/interface/InputTag.h"
@@ -45,14 +47,15 @@ void MillePedeFileExtractor::endLuminosityBlock(const edm::LuminosityBlock& iLum
       // formatting directive for a number, like %04d.
       char theNumberedOutputFileName[200];
       sprintf(theNumberedOutputFileName, outputFileName_.c_str(), nBinaries_);
+
       // Log the filename to which we will write...
       edm::LogInfo("MillePedeFileActions")
           << "Writing FileBlob file to file "
           << outputDir_ + theNumberedOutputFileName << ".";
+
       // ...and perform the writing operation.
-      blob.write(outputDir_ + theNumberedOutputFileName);
-      // Careful, it seems that when writing to an impossible file, this is
-      // swallowed by the FileBlob.write operation and no error is thrown.
+      writeGzipped(blob, outputDir_ + theNumberedOutputFileName);
+
       ++nBinaries_;
     }
   } else {
@@ -62,6 +65,30 @@ void MillePedeFileExtractor::endLuminosityBlock(const edm::LuminosityBlock& iLum
 }
 
 
+void MillePedeFileExtractor::writeGzipped(const FileBlob& blob,
+					  const std::string& fileName) {
+  // - use zlib directly to avoid boost dependencies for this simple task
+  // - zlib and gzip compression differ -> get uncompressed blob first
+  auto uncompressedBlob = blob.getUncompressedBlob();
+  gzFile fp = gzopen(fileName.c_str(), "wb");
+  if (fp == NULL) {
+    edm::LogError("MillePedeFileActions")
+      << "Problem while opening gzipped file '" << fileName << "'.";
+  }
+  auto nBytes = gzwrite(fp, &uncompressedBlob->front(), uncompressedBlob->size());
+  if (nBytes == 0 ||
+      nBytes != static_cast<decltype(nBytes)>(uncompressedBlob->size())) {
+    edm::LogError("MillePedeFileActions")
+      << "Problem while writing FileBlob to gzipped file '" << fileName << "'.";
+  }
+  auto zerr = gzclose(fp);
+  if (zerr != 0) {
+    edm::LogError("MillePedeFileActions")
+      << "Problem while closing gzipped file '" << fileName << "'.";
+  }
+}
+
+
 // Manage the parameters for the module:
 // (Note that this will autogenerate the _cfi.py file.)
 void MillePedeFileExtractor::fillDescriptions(

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeFileExtractor.h

@@ -46,6 +46,8 @@ class MillePedeFileExtractor :
     return (nBinaries_ >= maxNumberOfBinaries_) && hasBinaryNumberLimit(); }
   bool hasBinaryNumberLimit() { return maxNumberOfBinaries_ > -1; }
 
+  static void writeGzipped(const FileBlob&, const std::string&);
+
   const std::string outputDir_;
   const std::string outputFileName_;
 

Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py

@@ -22,6 +22,10 @@
     monitorFile = cms.untracked.string('millePedeMonitor.root'), ## if empty: no monitoring...
 
     runAtPCL = cms.bool(False), # at the PCL the mille binaries are reset at lumi-section boundaries
+    ignoreHitsWithoutGlobalDerivatives = cms.bool(False), # - if all alignables and calibration for a
+                                                          #   hit are set to '0', the hit is ignored
+                                                          # - has only an effect with non-GBL
+                                                          #   material-effects description
 
     # PSet that allows to configure the pede labeler, i.e. select the actual
     # labeler plugin to use and parameters for the selected plugin