Skip to content

Commit

Permalink
Merge pull request #16135 from ghellwig/temporary-fix_multi-IOV-input…
Browse files Browse the repository at this point in the history
…_MillePede_81X

[81X] Temporary fix for multi-IOV input in MillePede
  • Loading branch information
cmsbuild committed Oct 10, 2016
2 parents 6d73f73 + 6dac1d3 commit 994786a
Show file tree
Hide file tree
Showing 7 changed files with 251 additions and 11 deletions.
Expand Up @@ -34,6 +34,7 @@

SiPixelAliPedeAlignmentProducer.algoConfig = MillePedeAlignmentAlgorithm
SiPixelAliPedeAlignmentProducer.algoConfig.mode = 'pede'
SiPixelAliPedeAlignmentProducer.algoConfig.runAtPCL = True
SiPixelAliPedeAlignmentProducer.algoConfig.mergeBinaryFiles = [SiPixelAliMilleFileExtractor.outputBinaryFile.value()]
SiPixelAliPedeAlignmentProducer.algoConfig.binaryFile = ''
SiPixelAliPedeAlignmentProducer.algoConfig.TrajectoryFactory = cms.PSet(
Expand Down
Expand Up @@ -37,6 +37,7 @@

SiPixelAliPedeAlignmentProducer.algoConfig = MillePedeAlignmentAlgorithm
SiPixelAliPedeAlignmentProducer.algoConfig.mode = 'pede'
SiPixelAliPedeAlignmentProducer.algoConfig.runAtPCL = True
SiPixelAliPedeAlignmentProducer.algoConfig.mergeBinaryFiles = [SiPixelAliMilleFileExtractor.outputBinaryFile.value()]
SiPixelAliPedeAlignmentProducer.algoConfig.binaryFile = ''
SiPixelAliPedeAlignmentProducer.algoConfig.TrajectoryFactory = cms.PSet(
Expand Down
Expand Up @@ -41,6 +41,10 @@
#include "Alignment/MuonAlignment/interface/AlignableMuon.h"
#include "Alignment/CommonAlignment/interface/AlignableExtras.h"

#include "CondFormats/AlignmentRecord/interface/TrackerAlignmentRcd.h"
#include "CondFormats/AlignmentRecord/interface/TrackerAlignmentErrorExtendedRcd.h"
#include "CondFormats/AlignmentRecord/interface/TrackerSurfaceDeformationRcd.h"

#include "DataFormats/CLHEP/interface/AlgebraicObjects.h"
#include "DataFormats/TrackReco/interface/Track.h"
#include "DataFormats/SiStripDetId/interface/SiStripDetId.h"
Expand All @@ -67,7 +71,6 @@ typedef TransientTrackingRecHit::ConstRecHitPointer ConstRecHitPointer;
typedef TrajectoryFactoryBase::ReferenceTrajectoryCollection RefTrajColl;

// Includes for PXB survey
#include <iostream>
#include "Alignment/SurveyAnalysis/interface/SurveyPxbImage.h"
#include "Alignment/SurveyAnalysis/interface/SurveyPxbImageLocalFit.h"
#include "Alignment/SurveyAnalysis/interface/SurveyPxbImageReader.h"
Expand Down Expand Up @@ -124,6 +127,51 @@ void MillePedeAlignmentAlgorithm::initialize(const edm::EventSetup &setup,
<< "Running with AlignabeMuon not yet tested.";
}

// temporary fix to avoid corrupted database output
if (!runAtPCL_) {
const auto MIN_VAL{cond::timeTypeSpecs[cond::runnumber].beginValue};
const auto MAX_VAL{cond::timeTypeSpecs[cond::runnumber].endValue};
const auto& iov_alignments =
setup.get<TrackerAlignmentRcd>().validityInterval();
const auto& iov_surfaces =
setup.get<TrackerSurfaceDeformationRcd>().validityInterval();
const auto& iov_errors =
setup.get<TrackerAlignmentErrorExtendedRcd>().validityInterval();
if (iov_alignments.first().eventID().run() != MIN_VAL ||
iov_alignments.last().eventID().run() != MAX_VAL) {
throw cms::Exception("DatabaseError")
<< "@SUB=MillePedeAlignmentAlgorithm::initialize"
<< "\nTrying to apply " << setup.get<TrackerAlignmentRcd>().key().name()
<< " with multiple IOVs in tag.\n"
<< "Validity range is "
<< iov_alignments.first().eventID().run() << " - "
<< iov_alignments.last().eventID().run();
}
if (iov_surfaces.first().eventID().run() != MIN_VAL ||
iov_surfaces.last().eventID().run() != MAX_VAL) {
throw cms::Exception("DatabaseError")
<< "@SUB=MillePedeAlignmentAlgorithm::initialize"
<< "\nTrying to apply "
<< setup.get<TrackerSurfaceDeformationRcd>().key().name()
<< " with multiple IOVs in tag.\n"
<< "Validity range is "
<< iov_surfaces.first().eventID().run() << " - "
<< iov_surfaces.last().eventID().run();
}
if (iov_errors.first().eventID().run() != MIN_VAL ||
iov_errors.last().eventID().run() != MAX_VAL) {
throw cms::Exception("DatabaseError")
<< "@SUB=MillePedeAlignmentAlgorithm::initialize"
<< "\nTrying to apply "
<< setup.get<TrackerAlignmentErrorExtendedRcd>().key().name()
<< " with multiple IOVs in tag.\n"
<< "Validity range is "
<< iov_errors.first().eventID().run() << " - "
<< iov_errors.last().eventID().run();
}
}


//Retrieve tracker topology from geometry
edm::ESHandle<TrackerTopology> tTopoHandle;
setup.get<TrackerTopologyRcd>().get(tTopoHandle);
Expand Down Expand Up @@ -625,7 +673,9 @@ int MillePedeAlignmentAlgorithm::addGlobalData(const edm::EventSetup &setup, con
theDoubleBufferX.push_back(iValuesInd->first.first);
theDoubleBufferY.push_back(iValuesInd->first.second);
} else {
std::cerr << "MillePedeAlignmentAlgorithm::addGlobalData: Invalid label " << globalLabel << " <= 0 or > 2147483647" << std::endl;
edm::LogError("Alignment")
<< "@SUB=MillePedeAlignmentAlgorithm::addGlobalData"
<< "Invalid label " << globalLabel << " <= 0 or > 2147483647";
}
}
}
Expand Down Expand Up @@ -745,7 +795,9 @@ ::globalDerivativesHierarchy(const EventInfo &eventInfo,
globalDerivativesX.push_back(derivs[iSel][kLocalX] / thePedeSteer->cmsToPedeFactor(iSel));
globalDerivativesY.push_back(derivs[iSel][kLocalY] / thePedeSteer->cmsToPedeFactor(iSel));
} else {
std::cerr << "MillePedeAlignmentAlgorithm::globalDerivativesHierarchy: Invalid label " << globalLabel << " <= 0 or > 2147483647" << std::endl;
edm::LogError("Alignment")
<< "@SUB=MillePedeAlignmentAlgorithm::globalDerivativesHierarchy"
<< "Invalid label " << globalLabel << " <= 0 or > 2147483647";
}
}
}
Expand Down Expand Up @@ -1391,7 +1443,12 @@ void MillePedeAlignmentAlgorithm::addPxbSurvey(const edm::ParameterSet &pxbSurve
{
// do some printing, if requested
const bool doOutputOnStdout(pxbSurveyCfg.getParameter<bool>("doOutputOnStdout"));
if (doOutputOnStdout) std::cout << "# Output from addPxbSurvey follows below because doOutputOnStdout is set to True" << std::endl;
if (doOutputOnStdout) {
edm::LogInfo("Alignment")
<< "@SUB=MillePedeAlignmentAlgorithm::addPxbSurvey"
<< "# Output from addPxbSurvey follows below because "
<< "doOutputOnStdout is set to True";
}

// instantiate a dicer object
SurveyPxbDicer dicer(pxbSurveyCfg.getParameter<std::vector<edm::ParameterSet> >("toySurveyParameters"), pxbSurveyCfg.getParameter<unsigned int>("toySurveySeed"));
Expand All @@ -1405,7 +1462,11 @@ void MillePedeAlignmentAlgorithm::addPxbSurvey(const edm::ParameterSet &pxbSurve
// loop over photographs (=measurements) and perform the fit
for(std::vector<SurveyPxbImageLocalFit>::size_type i=0; i!=measurements.size(); i++)
{
if (doOutputOnStdout) std::cout << "Module " << i << ": ";
if (doOutputOnStdout) {
edm::LogInfo("Alignment")
<< "@SUB=MillePedeAlignmentAlgorithm::addPxbSurvey"
<< "Module " << i << ": ";
}

// get the Alignables and their surfaces
AlignableDetOrUnitPtr mod1(theAlignableNavigator->alignableFromDetId(measurements[i].getIdFirst()));
Expand Down Expand Up @@ -1449,10 +1510,12 @@ void MillePedeAlignmentAlgorithm::addPxbSurvey(const edm::ParameterSet &pxbSurve
// do some reporting, if requested
if (doOutputOnStdout)
{
std::cout << "a: " << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3]
<< " S= " << sqrt(a[2]*a[2]+a[3]*a[3])
<< " phi= " << atan(a[3]/a[2])
<< " chi2= " << chi2 << std::endl;
edm::LogInfo("Alignment")
<< "@SUB=MillePedeAlignmentAlgorithm::addPxbSurvey"
<< "a: " << a[0] << ", " << a[1] << ", " << a[2] << ", " << a[3]
<< " S= " << sqrt(a[2]*a[2]+a[3]*a[3])
<< " phi= " << atan(a[3]/a[2])
<< " chi2= " << chi2 << std::endl;
}
if (theMonitor)
{
Expand Down
73 changes: 73 additions & 0 deletions Alignment/MillePedeAlignmentAlgorithm/python/mpslib/tools.py
@@ -0,0 +1,73 @@
import os
import sys
import subprocess
import CondCore.Utilities.CondDBFW.shell as shell


def create_single_iov_db(global_tag, run_number, output_db):
"""Create an sqlite file with single-IOV tags for alignment payloads.
Arguments:
- `global_tag`: global tag from which to extract the payloads
- `run_number`: run for which the IOVs are selected
- `output_db`: name of the output sqlite file
"""

con = shell.connect()
tags = con.global_tag_map(global_tag_name = global_tag,
record = ["TrackerAlignmentRcd",
"TrackerSurfaceDeformationRcd",
"TrackerAlignmentErrorExtendedRcd"])
con.close_session()

tags = {item["record"]: {"name": item["tag_name"]}
for item in tags.as_dicts()}

for record,tag in tags.iteritems():
iovs = con.tag(name = tag["name"]).iovs().as_dicts()
run_is_covered = False
for iov in reversed(iovs):
if iov["since"] <= run_number:
tag["since"] = str(iov["since"])
run_is_covered = True
break
if not run_is_covered:
msg = ("Run number {0:d} is not covered in '{1:s}' ({2:s}) from"
" '{3:s}'.".format(run_number, tag["name"], record,
global_tag))
print msg
print "Aborting..."
sys.exit(1)

result = {}
if os.path.exists(output_db): os.remove(output_db)
for record,tag in tags.iteritems():
result[record] = {"connect": "sqlite_file:"+output_db,
"tag": "_".join([tag["name"], tag["since"]])}
cmd = ("conddb_import",
"-f", "frontier://PromptProd/cms_conditions",
"-c", result[record]["connect"],
"-i", tag["name"],
"-t", result[record]["tag"],
"-b", str(run_number),
"-e", str(run_number))
run_checked(cmd)
run_checked(["sqlite3", output_db, "update iov set since=1"])

return result


def run_checked(cmd):
"""Run `cmd` and exit in case of failures.
Arguments:
- `cmd`: list containing the strings of the command
"""

try:
with open(os.devnull, "w") as devnull:
subprocess.check_call(cmd, stdout = devnull)
except subprocess.CalledProcessError as e:
print "Problem in running the following command:"
print " ".join(e.cmd)
sys.exit(1)
64 changes: 62 additions & 2 deletions Alignment/MillePedeAlignmentAlgorithm/scripts/mps_alisetup.py
Expand Up @@ -12,12 +12,14 @@
import argparse
import os
import re
import importlib
import subprocess
import ConfigParser
import sys
import itertools
import collections
import Alignment.MillePedeAlignmentAlgorithm.mpslib.Mpslibclass as mpslib
import Alignment.MillePedeAlignmentAlgorithm.mpslib.tools as mps_tools
from Alignment.MillePedeAlignmentAlgorithm.alignmentsetup.helper import checked_out_MPS
from functools import reduce

Expand Down Expand Up @@ -74,6 +76,49 @@ def get_weight_configs(config):
return configs


def create_input_db(cfg, run_number):
"""
Create sqlite file with single-IOV tags and use it to override the GT. If
the GT is already customized by the user, the customization has higher
priority. Returns a snippet to be appended to the configuration file
Arguments:
- `cfg`: path to python configuration
- `run_number`: run from which to extract the alignment payloads
"""

sys.path.append(os.path.dirname(cfg))
cache_stdout = sys.stdout
sys.stdout = open(os.devnull, "w") # suppress unwanted output
__configuration = \
importlib.import_module(os.path.splitext(os.path.basename(cfg))[0])
sys.stdout = cache_stdout

run_number = int(run_number)
if not run_number > 0:
print "'FirstRunForStartGeometry' must be positive, but is", run_number
sys.exit(1)

global_tag = __configuration.process.GlobalTag.globaltag.value()
input_db_name = os.path.abspath("alignment_input.db")
tags = mps_tools.create_single_iov_db(global_tag, run_number, input_db_name)

for condition in __configuration.process.GlobalTag.toGet.value():
if condition.record.value() in tags: del tags[condition.record.value()]

result = ""
for record,tag in tags.iteritems():
if result == "":
result += ("\nimport "
"Alignment.MillePedeAlignmentAlgorithm.alignmentsetup."
"SetCondition as tagwriter\n")
result += ("\ntagwriter.setCondition(process,\n"
" connect = \""+tag["connect"]+"\",\n"
" record = \""+record+"\",\n"
" tag = \""+tag["tag"]+"\")\n")

os.remove(cfg+"c")
return result

# ------------------------------------------------------------------------------
# set up argument parser and config parser
Expand Down Expand Up @@ -141,7 +186,7 @@ def get_weight_configs(config):
generalOptions = {}

# essential variables
for var in ['classInf','pedeMem','jobname']:
for var in ["classInf","pedeMem","jobname", "FirstRunForStartGeometry"]:
try:
generalOptions[var] = config.get('general',var)
except ConfigParser.NoOptionError:
Expand Down Expand Up @@ -264,6 +309,7 @@ def get_weight_configs(config):
#------------------------------------------------------------------------------
# loop over dataset-sections
firstDataset = True
overrideGT = ""
for section in config.sections():
if 'general' in section:
continue
Expand Down Expand Up @@ -395,6 +441,11 @@ def get_weight_configs(config):
append = ''
firstDataset = False
configTemplate = tmpFile
overrideGT = create_input_db(thisCfgTemplate,
generalOptions["FirstRunForStartGeometry"])

with open(thisCfgTemplate, "a") as f: f.write(overrideGT)


# create mps_setup command
command = 'mps_setup.pl -m%s -M %s -N %s %s %s %s %d %s %s %s cmscafuser:%s' % (
Expand Down Expand Up @@ -473,7 +524,7 @@ def get_weight_configs(config):

thisCfgTemplate = "tmp.py"
with open(thisCfgTemplate, "w") as f:
f.write(configTemplate)
f.write(configTemplate+overrideGT)

# create new merge-config
command = ("mps_merge.py -w "+thisCfgTemplate+" jobData/"+lib.JOBDIR[-1]+
Expand All @@ -490,3 +541,12 @@ def get_weight_configs(config):

# remove temporary file
os.system("rm "+thisCfgTemplate)

if overrideGT.strip() != "":
print "="*60
msg = ("Overriding global tag with single-IOV tags extracted from '{}' for "
"run number '{}'.".format(generalOptions["globaltag"],
generalOptions["FirstRunForStartGeometry"]))
print msg
print "-"*60
print overrideGT
@@ -0,0 +1,41 @@
#!/usr/bin/env python

import sys
import argparse
import Alignment.MillePedeAlignmentAlgorithm.mpslib.tools as mps_tools

################################################################################
def main(argv = None):
"""Main routine of the script.
Arguments:
- `argv`: arguments passed to the main routine
"""

if argv == None:
argv = sys.argv[1:]

parser = argparse.ArgumentParser(
description="Prepare input db file for MillePede workflow.")
parser.add_argument("-g", "--global-tag", dest="global_tag", required=True,
metavar="TAG",
help="global tag to extract the alignment payloads")
parser.add_argument("-r", "--run-number", dest="run_number", required=True,
metavar="INTEGER", type=int,
help="run number to select IOV")
parser.add_argument("-o", "--output-db", dest="output_db",
default="alignment_input.db", metavar="PATH",
help="name of the output file (default: '%(default)s')")
args = parser.parse_args(argv)

mps_tools.create_single_iov_db(args.global_tag,
args.run_number,
args.output_db)


################################################################################
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
pass
Expand Up @@ -29,6 +29,7 @@ pedeMem = 32000
datasetdir = /afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/datasetfiles
configTemplate = universalConfigTemplate.py
globaltag = auto:run2_data
FirstRunForStartGeometry = 0 ; set this to the run from where you want to start

;####################################################################
;## datasets
Expand Down

0 comments on commit 994786a

Please sign in to comment.