[91X] Geometry-independent MillePede validation

Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeVariables.h

@@ -15,15 +15,16 @@
 
 #include "Alignment/CommonAlignment/interface/AlignmentUserVariables.h"
 
+#include <string>
 #include <vector>
 
 class MillePedeVariables : public AlignmentUserVariables {
  public:
 
   /** constructor */
-  MillePedeVariables(unsigned int nParams, unsigned int label);
+  MillePedeVariables(unsigned int nParams, unsigned int label, const std::string& name);
   /** destructor */
-  virtual ~MillePedeVariables() {}
+  virtual ~MillePedeVariables() = default;
   /** clone method (using copy constructor) */
   virtual MillePedeVariables* clone() const { return new MillePedeVariables(*this);}
 
@@ -67,6 +68,11 @@ class MillePedeVariables : public AlignmentUserVariables {
   /// set alignable label as used by pede
   void setLabel(unsigned int label) { myLabel = label;}
 
+  /// get alignable name
+  const std::string& name() const { return myName; }
+  /// set alignable name
+  void setName(const std::string& name) { myName = name;}
+
   /// get number of hits for x-measurement
   unsigned int hitsX() const {return myHitsX;}
   /// increase hits for x-measurement
@@ -94,6 +100,7 @@ class MillePedeVariables : public AlignmentUserVariables {
   unsigned int       myHitsX;
   unsigned int       myHitsY;
   unsigned int       myLabel;
+  std::string        myName;
 };
 
 #endif

Alignment/MillePedeAlignmentAlgorithm/interface/MillePedeVariablesIORoot.h

@@ -15,6 +15,7 @@
 #include "Alignment/CommonAlignmentAlgorithm/interface/AlignmentIORootBase.h"
 #include "Alignment/CommonAlignmentAlgorithm/interface/AlignmentUserVariablesIO.h"
 
+#include <string>
 #include <vector>
 
 // ROOT types:
@@ -78,6 +79,8 @@ class MillePedeVariablesIORoot : public AlignmentIORootBase, public AlignmentUse
   UInt_t       myHitsX;
   UInt_t       myHitsY;
   UInt_t       myLabel;
+  std::string  myName;
+  std::string* myNamePtr;	// needed for ROOT IO
 };
 
 #endif

Alignment/MillePedeAlignmentAlgorithm/plugins/MillePedeAlignmentAlgorithm.cc

@@ -1179,8 +1179,8 @@ unsigned int MillePedeAlignmentAlgorithm::doIO(int loop) const
 //__________________________________________________________________________________________________
 void MillePedeAlignmentAlgorithm::buildUserVariables(const std::vector<Alignable*> &alis) const
 {
-  for (std::vector<Alignable*>::const_iterator iAli = alis.begin(); iAli != alis.end(); ++iAli) {
-    AlignmentParameters *params = (*iAli)->alignmentParameters();
+  for (const auto& iAli: alis) {
+    AlignmentParameters *params = iAli->alignmentParameters();
     if (!params) {
       throw cms::Exception("Alignment") << "@SUB=MillePedeAlignmentAlgorithm::buildUserVariables"
                                         << "No parameters for alignable";
@@ -1195,7 +1195,8 @@ void MillePedeAlignmentAlgorithm::buildUserVariables(const std::vector<Alignable
         //std::cout << "\nAfter: " << userVars->parameter()[iPar] << std::endl;
       }
     } else { // Nothing yet or erase wrong type:
-      userVars = new MillePedeVariables(params->size(), thePedeLabels->alignableLabel(*iAli));
+      userVars = new MillePedeVariables(params->size(), thePedeLabels->alignableLabel(iAli),
+                                        thePedeLabels->alignableTracker()->objectIdProvider().typeToName(iAli->alignableObjectId()));
       params->setUserVariables(userVars);
     }
   }

Alignment/MillePedeAlignmentAlgorithm/python/mpslib/tools.py

@@ -36,9 +36,14 @@ def create_single_iov_db(inputs, run_number, output_db):
     for record,tag in inputs.iteritems():
         result[record] = {"connect": "sqlite_file:"+output_db,
                           "tag": "_".join([tag["tag"], tag["since"]])}
-        source_connect = ("frontier://PromptProd/cms_conditions"
-                          if tag["connect"] == "pro"
-                          else tag["connect"])
+
+        if tag["connect"] == "pro":
+            source_connect = "frontier://FrontierProd/CMS_CONDITIONS"
+        elif tag["connect"] == "dev":
+            source_connect = "frontier://FrontierPrep/CMS_CONDITIONS"
+        else:
+            source_connect = tag["connect"]
+
         cmd = ("conddb_import",
                "-f", source_connect,
                "-c", result[record]["connect"],
@@ -53,16 +58,20 @@ def create_single_iov_db(inputs, run_number, output_db):
     return result
 
 
-def run_checked(cmd):
+def run_checked(cmd, suppress_stderr = False):
     """Run `cmd` and exit in case of failures.
 
     Arguments:
     - `cmd`: list containing the strings of the command
+    - `suppress_stderr`: suppress output from stderr
     """
 
     try:
         with open(os.devnull, "w") as devnull:
-            subprocess.check_call(cmd, stdout = devnull)
+            if suppress_stderr:
+                subprocess.check_call(cmd, stdout = devnull, stderr = devnull)
+            else:
+                subprocess.check_call(cmd, stdout = devnull)
     except subprocess.CalledProcessError as e:
         print "Problem in running the following command:"
         print " ".join(e.cmd)
@@ -79,8 +88,12 @@ def get_process_object(cfg):
     sys.path.append(os.path.dirname(cfg)) # add location to python path
     cache_stdout = sys.stdout
     sys.stdout = open(os.devnull, "w")    # suppress unwanted output
-    __configuration = \
-        importlib.import_module(os.path.splitext(os.path.basename(cfg))[0])
+    try:
+        __configuration = \
+            importlib.import_module(os.path.splitext(os.path.basename(cfg))[0])
+    except Exception as e:
+        print "Problem detected in configuration file '{0}'.".format(cfg)
+        raise e
     sys.stdout = cache_stdout
     sys.path.pop()                        # clean up python path again
     try:
@@ -155,6 +168,7 @@ def get_iovs(db, tag):
 
     db = db.replace("sqlite_file:", "").replace("sqlite:", "")
     db = db.replace("frontier://FrontierProd/CMS_CONDITIONS", "pro")
+    db = db.replace("frontier://FrontierPrep/CMS_CONDITIONS", "dev")
 
     con = conddb.connect(url = conddb.make_url(db))
     session = con.session()

Alignment/MillePedeAlignmentAlgorithm/python/mpsvalidate/additionalparser.py

@@ -1,102 +1,50 @@
-#!/usr/bin/env python
-
 ##########################################################################
 # Parse the alignment_merge.py file for additional information
 #
 
 import logging
+import Alignment.MillePedeAlignmentAlgorithm.mpslib.tools as mps_tools
 
 
 class AdditionalData:
     """ stores the additional information of the alignment_merge.py file
     """
 
     def __init__(self):
-        self.pedeSteererMethod = ""
-        self.pedeSteererOptions = []
-        self.pedeSteererCommand = ""
+        self.pede_steerer_method  = ""
+        self.pede_steerer_options = []
+        self.pede_steerer_command = ""
 
-        # safe the selector information Rigid, Bowed, TwoBowed
-        self.selector = [[] for x in range(3)]
-        self.selectorTag = [[] for x in range(3)]
-        self.selectorThird = [[] for x in range(3)]
+        self.selectors = {}
+        self.iov_definition = ""
 
-        # string to find the information and variables where to safe the
-        # information (searchstring: [selector list, seletor tag, third element,
-        # name])
-        self.pattern = {
-            "process.AlignmentProducer.ParameterBuilder.SelectorRigid = cms.PSet(": [self.selector[0], self.selectorTag[0], self.selectorThird[0], "SelectorRigid"],
-            "process.AlignmentProducer.ParameterBuilder.SelectorBowed = cms.PSet(": [self.selector[1], self.selectorTag[1], self.selectorThird[1], "SelectorBowed"],
-            "process.AlignmentProducer.ParameterBuilder.SelectorTwoBowed = cms.PSet(": [self.selector[2], self.selectorTag[2], self.selectorThird[2], "SelectorTwoBowed"]
-        }
 
     def parse(self, config, path):
         logger = logging.getLogger("mpsvalidate")
-        
-        # open aligment_merge.py file
+
+        # extract process object from aligment_merge.py file
         try:
-            with open(path) as inputFile:
-                mergeFile = inputFile.readlines()
-        except IOError:
+            process = mps_tools.get_process_object(path)
+        except ImportError:
             logger.error("AdditionalData: {0} does not exist".format(path))
             return
 
-        # search pattern
-
-        # loop over lines
-        for index, line in enumerate(mergeFile):
-            try:
-                # search for SelectorRigid, SelectorBowed and SelectorTwoBowed
-                for string in self.pattern:
-                    if (string in line):
-                        # extract data
-                        for lineNumber in range(index + 2, index + 8):
-                            mergeFile[lineNumber] = mergeFile[lineNumber].split("#", 1)[0]
-                            # break at the end of the SelectorRigid
-                            if (")" in mergeFile[lineNumber]):
-                                break
-                            self.pattern[string][0].append(
-                                mergeFile[lineNumber].replace("\"", "'").strip("', \n").split(","))
-                            # check if third argument
-                            if (len(self.pattern[string][0][-1]) > 2):
-                                self.pattern[string][1].append(
-                                    self.pattern[string][0][-1][2])
-                    # check for third arguments
-                    if ("'" not in line.replace("\"", "'")):
-                        for tag in self.pattern[string][1]:
-                            if tag in line:
-                                self.pattern[string][2].append(line.strip("\n").replace("#", ""))
-                                # add following lines
-                                for lineNumber in range(index + 1, index + 5):
-                                    # new process or blank line
-                                    if ("process" in mergeFile[lineNumber] or "\n" == mergeFile[lineNumber]):
-                                        break
-                                    # different tag
-                                    if (any(x in mergeFile[lineNumber] for x in self.pattern[string][1])):
-                                        break
-                                    self.pattern[string][2].append(mergeFile[lineNumber].strip("\n").replace("#", ""))
-            except Exception as e:
-                logging.error("Selector Parsing error")
-
-            # search for pedeSteererMethod
-            if ("process.AlignmentProducer.algoConfig.pedeSteerer.method" in line and "#" not in line):
-                try:
-                    self.pedeSteererMethod = line.replace("\"", "'").split("'")[1]
-                except Exception as e:
-                    logger.error("AdditionalParser: pedeSteererMethod not found - {0}".format(e))
-
-            # search for pedeSteererOptions
-            if ("process.AlignmentProducer.algoConfig.pedeSteerer.options" in line and "#" not in line):
-                for lineNumber in range(index + 1, index + 15):
-                    if (lineNumber<len(mergeFile)):
-                        if ("]" in mergeFile[lineNumber]):
-                            break
-                        self.pedeSteererOptions.append(
-                            mergeFile[lineNumber].replace("\"", "'").strip("', \n"))
-
-            # search for pedeSteererCommand
-            if ("process.AlignmentProducer.algoConfig.pedeSteerer.pedeCommand" in line and "#" not in line):
-                try:
-                    self.pedeSteererCommand = line.replace("\"", "'").split("'")[1]
-                except Exception as e:
-                    logger.error("AdditionalParser: pedeSteererCommand not found - {0}".format(e))
+        # find alignable selectors
+        param_builder = process.AlignmentProducer.ParameterBuilder
+        for index,sel in enumerate(param_builder.parameterTypes):
+            selector_name = sel.split(",")[0].strip()
+            self.selectors[index] = {
+                "name": selector_name,
+                "selector": getattr(param_builder, selector_name),
+                }
+
+        # find IOV definition
+        if len(process.AlignmentProducer.RunRangeSelection) > 0:
+            self.iov_definition = \
+                process.AlignmentProducer.RunRangeSelection.dumpPython()
+
+        # find pede steerer configuration
+        pede_steerer = process.AlignmentProducer.algoConfig.pedeSteerer
+        self.pede_steerer_method  = pede_steerer.method.value()
+        self.pede_steerer_options = pede_steerer.options.value()
+        self.pede_steerer_command = pede_steerer.pedeCommand.value()

Alignment/MillePedeAlignmentAlgorithm/python/mpsvalidate/beamerCreator.py

@@ -1,15 +1,11 @@
-#!/usr/bin/env python
-
 ##########################################################################
 # Creates beamer out of the histograms, parsed data and a given template.
 ##
 
 import logging
 import os
 import string
-
-from Alignment.MillePedeAlignmentAlgorithm.mpsvalidate.classes import MonitorData, PedeDumpData
-from Alignment.MillePedeAlignmentAlgorithm.mpsvalidate.geometry import Alignables, Structure
+import Alignment.MillePedeAlignmentAlgorithm.mpsvalidate.classes as mpsv_classes
 
 
 # create class to have delimiter %% which is not used in latex
@@ -48,7 +44,8 @@ def create(alignables, pedeDump, additionalData, outputFile, config):
     logger = logging.getLogger("mpsvalidate")
 
     # load template
-    with open(os.path.join(config.mpspath, "beamer_template.tex"), "r") as template:
+    with open(os.path.join(config.mpspath, "templates",
+                           "mpsvalidate_beamer_template.tex")) as template:
         data = template.read()
         template.close()
 
@@ -95,33 +92,34 @@ def create(alignables, pedeDump, additionalData, outputFile, config):
     try:
         out.add("\subsection{Alignment Configuration}")
         text = "\\textbf{{PedeSteerer method:}} {{{0}}}\\\\\n".format(
-            additionalData.pedeSteererMethod)
+            additionalData.pede_steerer_method)
         text += "\\textbf{{PedeSteerer options:}}\\\\\n"
-        for line in additionalData.pedeSteererOptions:
+        for line in additionalData.pede_steerer_options:
             text += "{{{0}}}\\\\\n".format(line)
         text += "\\textbf{{PedeSteerer command:}} {0}\\\\\n".format(
-            additionalData.pedeSteererCommand)
+            additionalData.pede_steerer_command)
         out.addSlide("Alignment Configuration", text)
     except Exception as e:
         logger.error("data not found - {0} {1}".format(type(e), e))
 
     # table of input files with number of tracks
-    if (config.showmonitor):
+    if config.showmonitor:
         out.add("\subsection{Datasets with tracks}")
         text = """\\begin{table}[h]
             \centering
             \caption{Datasets with tracks}
-            \\begin{tabular}{cc}
+            \\begin{tabular}{ccc}
             \hline
-            Dataset & Number of used tracks \\\\
+            Dataset & Number of used tracks & Weight \\\\
             \hline \n"""
         try:
-            for monitor in MonitorData.monitors:
-                text += "{0} & {1}\\\\\n".format(monitor.name, monitor.ntracks)
+            for monitor in mpsv_classes.MonitorData.monitors:
+                text += "{0} & {1} & {2}\\\\\n".format(monitor.name, monitor.ntracks,
+                                                       monitor.weight if monitor.weight != None else "--")
         except Exception as e:
             logger.error("data not found - {0} {1}".format(type(e), e))
         if (pedeDump.nrec):
-            text += "Number of records & {0}\\\\\n".format(pedeDump.nrec)
+            text += "\hline\nNumber of records & {0}\\\\\n".format(pedeDump.nrec)
         text += """\hline
                   \end{tabular}\n
                   \end{table}\n"""