Merge pull request #17408 from ghellwig/MPS_updates_90X

[90X] Update of the Millepede Production System

Alignment/CommonAlignment/python/tools/trackselectionRefitting.py

@@ -153,7 +153,27 @@ def getSequence(process, collection,
         options["TrackHitFilter"]["Tracker"].update({
                 "minimumHits": 10,
                 })
-        pass
+    elif collection == "ALCARECOTkAlUpsilonMuMu":
+        options["TrackSelector"]["Alignment"].update({
+                "ptMin": 3.0,
+                "etaMin": -2.4,
+                "etaMax": 2.4,
+                "nHitMin": 10,
+                "applyMultiplicityFilter": True,
+                "minMultiplicity": 2,
+                "maxMultiplicity": 2,
+                ("minHitsPerSubDet", "inPIXEL"): 1,
+                ("TwoBodyDecaySelector", "applyChargeFilter"): True,
+                ("TwoBodyDecaySelector", "charge"): 0,
+                ("TwoBodyDecaySelector",
+                 "applyMassrangeFilter"): not openMassWindow,
+                ("TwoBodyDecaySelector", "minXMass"): 9.2,
+                ("TwoBodyDecaySelector", "maxXMass"): 9.7,
+                ("TwoBodyDecaySelector", "daughterMass"): 0.105
+                })
+        options["TrackHitFilter"]["Tracker"].update({
+                "minimumHits": 10,
+                })
     else:
         print "Unknown input track collection:", collection
         sys.exit(1)

Alignment/CommonAlignmentAlgorithm/interface/AlignmentAlgorithmBase.h

@@ -149,7 +149,7 @@ class AlignmentAlgorithmBase
   virtual void run( const edm::EventSetup &setup, const EventInfo &eventInfo) = 0;
 
   /// called at begin of run
-  virtual void beginRun(const edm::EventSetup &setup) {};
+  virtual void beginRun(const edm::Run& run, const edm::EventSetup &setup) {};
 
   /// called at end of run - order of arguments like in EDProducer etc.
   virtual void endRun(const EndRunInfo &runInfo, const edm::EventSetup &setup) {};

Alignment/CommonAlignmentProducer/plugins/AlignmentProducer.cc

@@ -81,6 +81,7 @@ AlignmentProducer::AlignmentProducer(const edm::ParameterSet& iConfig) :
   theAlignmentAlgo(0), theAlignmentParameterStore(0),
   theAlignableExtras(0), theAlignableTracker(0), theAlignableMuon(0), 
   globalPositions_(0),
+  uniqueRunRanges_(makeUniqueRunRanges(iConfig)),
   nevent_(0), theParameterSet(iConfig),
   theMaxLoops( iConfig.getUntrackedParameter<unsigned int>("maxLoops") ),
   stNFixAlignables_(iConfig.getParameter<int>("nFixAlignables") ),
@@ -116,6 +117,7 @@ AlignmentProducer::AlignmentProducer(const edm::ParameterSet& iConfig) :
   edm::ParameterSet algoConfig = iConfig.getParameter<edm::ParameterSet>( "algoConfig" );
   edm::VParameterSet iovSelection = iConfig.getParameter<edm::VParameterSet>( "RunRangeSelection" );
   algoConfig.addUntrackedParameter<edm::VParameterSet>( "RunRangeSelection", iovSelection );
+  algoConfig.addUntrackedParameter<RunNumber>("firstIOV", uniqueRunRanges_.front().first);
   std::string algoName = algoConfig.getParameter<std::string>( "algoName" );
   theAlignmentAlgo = AlignmentAlgorithmPluginFactory::get( )->create( algoName, algoConfig  );
 
@@ -333,53 +335,41 @@ void AlignmentProducer::endOfJob()
     edm::LogError("Alignment") << "@SUB=AlignmentProducer::endOfJob" << "Did not process any "
                                << "events in last loop, do not dare to store to DB.";
   } else {
-
-    // Expand run ranges and make them unique
-    edm::VParameterSet runRangeSelectionVPSet(theParameterSet.getParameter<edm::VParameterSet>("RunRangeSelection"));
-    RunRanges uniqueRunRanges(this->makeNonOverlappingRunRanges(runRangeSelectionVPSet));
-    if (uniqueRunRanges.empty()) { // create dummy IOV
-      const RunRange runRange(cond::timeTypeSpecs[cond::runnumber].beginValue,
-			      cond::timeTypeSpecs[cond::runnumber].endValue);
-      uniqueRunRanges.push_back(runRange);
-    }
-
     std::vector<AlgebraicVector> beamSpotParameters;
 
-    for (RunRanges::const_iterator iRunRange = uniqueRunRanges.begin();
-	 iRunRange != uniqueRunRanges.end();
-	 ++iRunRange) {
+    for (const auto& iRunRange: uniqueRunRanges_) {
 
-      theAlignmentAlgo->setParametersForRunRange(*iRunRange);
+      theAlignmentAlgo->setParametersForRunRange(iRunRange);
 
       // Save alignments to database
       if (saveToDB_ || saveApeToDB_ || saveDeformationsToDB_)
-        this->writeForRunRange((*iRunRange).first);
+        this->writeForRunRange(iRunRange.first);
 
       // Deal with extra alignables, e.g. beam spot
       if (theAlignableExtras) {
-	Alignables &alis = theAlignableExtras->beamSpot();
-	if (!alis.empty()) {
-	  BeamSpotAlignmentParameters *beamSpotAliPars = dynamic_cast<BeamSpotAlignmentParameters*>(alis[0]->alignmentParameters());
-	  beamSpotParameters.push_back(beamSpotAliPars->parameters());
-	}
+        Alignables &alis = theAlignableExtras->beamSpot();
+        if (!alis.empty()) {
+          BeamSpotAlignmentParameters *beamSpotAliPars = dynamic_cast<BeamSpotAlignmentParameters*>(alis[0]->alignmentParameters());
+          beamSpotParameters.push_back(beamSpotAliPars->parameters());
+        }
       }
     }
 
     if (theAlignableExtras) {
       std::ostringstream bsOutput;
 
       std::vector<AlgebraicVector>::const_iterator itPar = beamSpotParameters.begin();
-      for (RunRanges::const_iterator iRunRange = uniqueRunRanges.begin();
-	   iRunRange != uniqueRunRanges.end();
-	   ++iRunRange, ++itPar) {
-	bsOutput << "Run range: " << (*iRunRange).first << " - " << (*iRunRange).second << "\n";
-	bsOutput << "  Displacement: x=" << (*itPar)[0] << ", y=" << (*itPar)[1] << "\n"; 
-	bsOutput << "  Slope: dx/dz=" << (*itPar)[2] << ", dy/dz=" << (*itPar)[3] << "\n"; 
+      for (auto iRunRange = uniqueRunRanges_.cbegin();
+           iRunRange != uniqueRunRanges_.cend();
+           ++iRunRange, ++itPar) {
+        bsOutput << "Run range: " << (*iRunRange).first << " - " << (*iRunRange).second << "\n";
+        bsOutput << "  Displacement: x=" << (*itPar)[0] << ", y=" << (*itPar)[1] << "\n";
+        bsOutput << "  Slope: dx/dz=" << (*itPar)[2] << ", dy/dz=" << (*itPar)[3] << "\n";
       }
 
       edm::LogInfo("Alignment") << "@SUB=AlignmentProducer::endOfJob"
-				<< "Parameters for alignable beamspot:\n"
-				<< bsOutput.str();
+                                << "Parameters for alignable beamspot:\n"
+                                << bsOutput.str();
     }
 
     for (auto iCal = theCalibrations.begin(); iCal != theCalibrations.end(); ++iCal) {
@@ -536,7 +526,7 @@ AlignmentProducer::duringLoop( const edm::Event& event,
 // ----------------------------------------------------------------------------
 void AlignmentProducer::beginRun(const edm::Run &run, const edm::EventSetup &setup)
 {
-  theAlignmentAlgo->beginRun(setup); // do not forward edm::Run...
+  theAlignmentAlgo->beginRun(run, setup);
 }
 
 // ----------------------------------------------------------------------------
@@ -1032,4 +1022,19 @@ AlignmentProducer::makeNonOverlappingRunRanges(const edm::VParameterSet& RunRang
   return uniqueRunRanges;
 }
 
+
+AlignmentProducer::RunRanges
+AlignmentProducer::makeUniqueRunRanges(const edm::ParameterSet& cfg) {
+  const auto runRangeSelectionVPSet =
+    cfg.getParameter<edm::VParameterSet>("RunRangeSelection");
+  auto uniqueRunRanges = makeNonOverlappingRunRanges(runRangeSelectionVPSet);
+  if (uniqueRunRanges.empty()) { // create dummy IOV
+    const RunRange runRange(cond::timeTypeSpecs[cond::runnumber].beginValue,
+			    cond::timeTypeSpecs[cond::runnumber].endValue);
+    uniqueRunRanges.push_back(runRange);
+  }
+  return uniqueRunRanges;
+}
+
+
 DEFINE_FWK_LOOPER( AlignmentProducer );

Alignment/CommonAlignmentProducer/plugins/AlignmentProducer.h

@@ -148,6 +148,7 @@ class AlignmentProducer : public edm::ESProducerLooper
   void readInSurveyRcds( const edm::EventSetup& );
 
   RunRanges makeNonOverlappingRunRanges(const edm::VParameterSet& RunRangeSelectionVPSet);
+  RunRanges makeUniqueRunRanges(const edm::ParameterSet& cfg);
 
   // private data members
 
@@ -170,6 +171,7 @@ class AlignmentProducer : public edm::ESProducerLooper
   /// GlobalPositions that might be read from DB, NULL otherwise
   const Alignments *globalPositions_;
 
+  const RunRanges uniqueRunRanges_;
   int nevent_;
   edm::ParameterSet theParameterSet;
 

Alignment/CommonAlignmentProducer/plugins/PCLTrackerAlProducer.cc

@@ -182,8 +182,7 @@ ::beginRun(const edm::Run& run, const edm::EventSetup& setup)
     initAlignmentAlgorithm(setup);
   }
 
-  // Do not forward edm::Run
-  theAlignmentAlgo->beginRun(setup);
+  theAlignmentAlgo->beginRun(run, setup);
 
   if (setupChanged(setup)) {
     initAlignmentAlgorithm(setup);
@@ -321,6 +320,8 @@ ::createAlignmentAlgorithm(const edm::ParameterSet& config)
   edm::ParameterSet algoConfig    = config.getParameter<edm::ParameterSet>("algoConfig");
   edm::VParameterSet iovSelection = config.getParameter<edm::VParameterSet>("RunRangeSelection");
   algoConfig.addUntrackedParameter<edm::VParameterSet>("RunRangeSelection", iovSelection);
+  // provide required parameter while keeping current functionality for PCL:
+  algoConfig.addUntrackedParameter<RunNumber>("firstIOV", 1);
 
   std::string algoName = algoConfig.getParameter<std::string>("algoName");
   theAlignmentAlgo = AlignmentAlgorithmPluginFactory::get()->create(algoName, algoConfig);

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.cc

@@ -12,6 +12,7 @@
 
 #include "FWCore/Framework/interface/ESHandle.h"
 #include "FWCore/Framework/interface/EventSetup.h"
+#include "FWCore/Framework/interface/Run.h"
 #include "FWCore/MessageLogger/interface/MessageLogger.h"
 
 #include "TrackingTools/PatternTools/interface/Trajectory.h"
@@ -95,6 +96,8 @@ MillePedeAlignmentAlgorithm::MillePedeAlignmentAlgorithm(const edm::ParameterSet
   theAlignables(),
   theMinNumHits(cfg.getParameter<unsigned int>("minNumHits")),
   theMaximalCor2D(cfg.getParameter<double>("max2Dcorrelation")),
+  firstIOV_(cfg.getUntrackedParameter<AlignmentAlgorithmBase::RunNumber>("firstIOV")),
+  ignoreFirstIOVCheck_(cfg.getUntrackedParameter<bool>("ignoreFirstIOVCheck")),
   theLastWrittenIov(0),
   theGblDoubleBinary(cfg.getParameter<bool>("doubleBinary")),
   runAtPCL_(cfg.getParameter<bool>("runAtPCL")),
@@ -582,6 +585,17 @@ MillePedeAlignmentAlgorithm::addHitCount(const std::vector<AlignmentParameters*>
   return nHitY;
 }
 
+
+void MillePedeAlignmentAlgorithm::beginRun(const edm::Run& run,
+                                           const edm::EventSetup& setup) {
+  if (run.run() < firstIOV_ && !ignoreFirstIOVCheck_) {
+    throw cms::Exception("Alignment")
+      << "@SUB=MillePedeAlignmentAlgorithm::beginRun\n"
+      << "Using data (run = " << run.run()
+      << ") prior to the first defined IOV (" << firstIOV_ << ").";
+  }
+}
+
 //____________________________________________________
 void MillePedeAlignmentAlgorithm::endRun(const EventInfo &eventInfo, const EndRunInfo &runInfo,
 					 const edm::EventSetup &setup)

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.h

@@ -86,6 +86,9 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase
   /// Run the algorithm on trajectories and tracks
   virtual void run(const edm::EventSetup &setup, const EventInfo &eventInfo) override;
 
+  /// called at begin of run
+  virtual void beginRun(const edm::Run& run, const edm::EventSetup& setup) override;
+
   // TODO: This method does NOT match endRun() in base class! Nobody is
   //       calling this?
   /// Run on run products, e.g. TkLAS
@@ -257,6 +260,8 @@ class MillePedeAlignmentAlgorithm : public AlignmentAlgorithmBase
   unsigned int              theMinNumHits;
   double                    theMaximalCor2D; /// maximal correlation allowed for 2D hit in TID/TEC.
                                              /// If larger, the 2D measurement gets diagonalized!!!
+  const AlignmentAlgorithmBase::RunNumber firstIOV_;
+  const bool ignoreFirstIOVCheck_;
   int                       theLastWrittenIov; // keeping track for output trees...
   std::vector<float>        theFloatBufferX;
   std::vector<float>        theFloatBufferY;

Alignment/MillePedeAlignmentAlgorithm/python/MillePedeAlignmentAlgorithm_cfi.py

@@ -23,6 +23,7 @@
     monitorFile = cms.untracked.string('millePedeMonitor.root'), ## if empty: no monitoring...
 
     runAtPCL = cms.bool(False), # at the PCL the mille binaries are reset at lumi-section boundaries
+    ignoreFirstIOVCheck = cms.untracked.bool(False), # flag to ignore check if data is prior to first IOV
     ignoreHitsWithoutGlobalDerivatives = cms.bool(False), # - if all alignables and calibration for a
                                                           #   hit are set to '0', the hit is ignored
                                                           # - has only an effect with non-GBL

Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/ConfigureAlignmentProducer.py

@@ -21,6 +21,22 @@ def setConfiguration(process, collection, mode, monitorFile, binaryFile,
     process.AlignmentProducer.algoConfig.mode              = mode
     process.AlignmentProducer.algoConfig.mergeBinaryFiles  = cms.vstring()
 
+    # default pede options:
+    process.AlignmentProducer.algoConfig.pedeSteerer.method = "sparseMINRES-QLP 3  0.8"
+    process.AlignmentProducer.algoConfig.minNumHits = 8
+    process.AlignmentProducer.algoConfig.pedeSteerer.options = [
+        "entries 50 10 2",
+        "outlierdownweighting 3",
+        "dwfractioncut 0.1",
+        "compress",
+        "threads 10 10",
+        "matiter 1",
+        "printcounts 2",
+        "chisqcut  30.  6.",
+        "bandwidth 6 1",
+        "monitorresiduals",
+    ]
+
     if mode == "mille":
         process.AlignmentProducer.algoConfig.binaryFile   = binaryFile
         process.AlignmentProducer.algoConfig.monitorFile  = monitorFile
@@ -43,6 +59,15 @@ def setConfiguration(process, collection, mode, monitorFile, binaryFile,
         process.AlignmentProducer.algoConfig.TrajectoryFactory.MaterialEffects = "LocalGBL"
         # to account for multiple scattering in these layers
         process.AlignmentProducer.algoConfig.TrajectoryFactory.UseInvalidHits = True
+    elif collection == "ALCARECOTkAlUpsilonMuMu":
+        process.AlignmentProducer.algoConfig.TrajectoryFactory = cms.PSet(
+             process.TwoBodyDecayTrajectoryFactory
+        )
+        process.AlignmentProducer.algoConfig.TrajectoryFactory.ParticleProperties.PrimaryMass =  9.4502
+        process.AlignmentProducer.algoConfig.TrajectoryFactory.ParticleProperties.PrimaryWidth = 0.0644
+        process.AlignmentProducer.algoConfig.TrajectoryFactory.MaterialEffects = "LocalGBL"
+        # to account for multiple scattering in these layers
+        process.AlignmentProducer.algoConfig.TrajectoryFactory.UseInvalidHits = True
     elif collection == "ALCARECOTkAlCosmicsCTF0T" and cosmicsZeroTesla:
         process.AlignmentProducer.algoConfig.TrajectoryFactory = cms.PSet(
             process.BrokenLinesBzeroTrajectoryFactory

Alignment/MillePedeAlignmentAlgorithm/python/alignmentsetup/helper.py

@@ -19,3 +19,23 @@ def checked_out_MPS():
         if e.args != (2, 'No such file or directory'): raise
 
     return checked_out, git_initialized
+
+
+def set_pede_option(process, option):
+    """Utility function to set or override pede `option` defined in `process`.
+
+    Arguments:
+    - `process`: cms.Process object
+    - `option`: option string
+    """
+
+    existing_options = process.AlignmentProducer.algoConfig.pedeSteerer.options
+
+    exists = False
+    for i in xrange(len(existing_options)):
+        if existing_options[i].split()[0] == option.split()[0]:
+           existing_options[i] = option.strip()
+           exists = True
+           break
+
+    if not exists: existing_options.append(option.strip())

Alignment/MillePedeAlignmentAlgorithm/python/mpslib/Mpslibclass.py

@@ -99,18 +99,18 @@ def read_db(self, db_file_name = __default_db_file_name):
             line = line.rstrip('\n')                #removes the pesky \n from line
             parts = line.split(":")                 #read each line and split into parts list
             self.JOBNUMBER.append(int(parts[0]))
-            self.JOBDIR.append(parts[1])
+            self.JOBDIR.append(parts[1].strip())
             self.JOBID.append(int(parts[2]))
-            self.JOBSTATUS.append(parts[3])
+            self.JOBSTATUS.append(parts[3].strip())
             self.JOBNTRY.append(int(parts[4]))
             self.JOBRUNTIME.append(int(parts[5]))   #int float?
             self.JOBNEVT.append(int(parts[6]))
-            self.JOBHOST.append(parts[7])
+            self.JOBHOST.append(parts[7].strip())
             self.JOBINCR.append(int(parts[8]))
-            self.JOBREMARK.append(parts[9])
-            self.JOBSP1.append(parts[10])
-            self.JOBSP2.append(parts[11])
-            self.JOBSP3.append(parts[12])
+            self.JOBREMARK.append(parts[9].strip())
+            self.JOBSP1.append(parts[10].strip())
+            self.JOBSP2.append(parts[11].strip())
+            self.JOBSP3.append(parts[12].strip())
 
             #count number of jobs
             if not self.JOBDIR[self.nJobs].startswith("jobm"):

Alignment/MillePedeAlignmentAlgorithm/python/mpsvalidate/.gitignore

@@ -0,0 +1 @@
+TrackerTree.root

Alignment/MillePedeAlignmentAlgorithm/scripts/mps_alisetup.py

@@ -246,6 +246,21 @@ def create_input_db(cfg, run_number):
         print "When using -w, a default configTemplate is needed to build a merge-config."
         raise SystemExit
 
+    try:
+        first_run = config.get("general", "FirstRunForStartGeometry")
+    except ConfigParser.NoOptionError:
+        print "Missing mandatory option 'FirstRunForStartGeometry' in [general] section."
+        raise SystemExit
+
+    for section in config.sections():
+        if section.startswith("dataset:"):
+            try:
+                collection = config.get(section, "collection")
+                break
+            except ConfigParser.NoOptionError:
+                print "Missing mandatory option 'collection' in section ["+section+"]."
+                raise SystemExit
+
     try:
         with open(configTemplate,"r") as f:
             tmpFile = f.read()
@@ -256,10 +271,14 @@ def create_input_db(cfg, run_number):
     tmpFile = re.sub('setupGlobaltag\s*\=\s*[\"\'](.*?)[\"\']',
                      'setupGlobaltag = \"'+globalTag+'\"',
                      tmpFile)
+    tmpFile = re.sub('setupCollection\s*\=\s*[\"\'](.*?)[\"\']',
+                     'setupCollection = \"'+collection+'\"',
+                     tmpFile)
 
     thisCfgTemplate = "tmp.py"
-    with open(thisCfgTemplate, "w") as f:
-        f.write(tmpFile)
+    with open(thisCfgTemplate, "w") as f: f.write(tmpFile)
+    overrideGT = create_input_db(thisCfgTemplate, first_run)
+    with open(thisCfgTemplate, "a") as f: f.write(overrideGT)
 
     for setting in pedesettings:
         print
@@ -303,6 +322,15 @@ def create_input_db(cfg, run_number):
 
     # remove temporary file
     os.system("rm "+thisCfgTemplate)
+    if overrideGT.strip() != "":
+        print "="*60
+        msg = ("Overriding global tag with single-IOV tags extracted from '{}' "
+               "for run number '{}'.".format(generalOptions["globaltag"],
+                                             first_run))
+        print msg
+        print "-"*60
+        print overrideGT
+
     sys.exit()